diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..281ea5aacce5661b1876f0521ff799146c91ea4a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,36 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_1_step_019799_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_5_step_003300_sample00.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_3_step_000000.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_0_step_009900_sample02.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_4_step_019799_sample01.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_0_step_019799_sample01.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_3_step_013200_sample01.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_1_step_013200.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_0_step_000000_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_0_step_009900_sample04.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/val_1_step_000000_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_5_step_016500_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_0_step_019799_sample00.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_3_step_019799_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_0_step_003300.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_3_step_003300_sample02.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_1_step_019799.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_2_step_003300_sample04.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_0_step_006600_sample01.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_5_step_000000_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/val_1_step_019799_sample04.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_0_step_006600_sample02.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_2_step_006600_sample00.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/val_0_step_013200_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_0_step_009900_sample00.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_0_step_009900_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_4_step_000000_sample04.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_2_step_013200_sample04.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/train_1_step_000000_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_2_step_013200_sample00.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/val_1_step_003300_sample03.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/test_3_step_013200_sample00.png filter=lfs diff=lfs merge=lfs -text +lora_256_dim_512_H100/step_vis/val_0_step_019799_sample04.png filter=lfs diff=lfs merge=lfs -text diff --git a/lora_128_dim_512_H100/train.log b/lora_128_dim_512_H100/train.log new file mode 100644 index 0000000000000000000000000000000000000000..cfc5469abe14123e51ac878b28c15cb28a5c7e9f --- /dev/null +++ b/lora_128_dim_512_H100/train.log @@ -0,0 +1,4400 @@ + +====================================================================== +Command: train_script.py --name lora_128_dim_512_H100 --batch-size 6 --adapter-lr 1e-3 --num-epochs 300 --lora-lr 1e-4 --adapter-mlp --lora-rank 128 --fusion-dim 512 --seed 42 +====================================================================== +Run dir : output/lora_128_dim_512_H100 +Log file: output/lora_128_dim_512_H100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (1.3s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260428_190804-eo9bzlri +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_128_dim_512_H100-train-20260428-190804 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100/runs/eo9bzlri + +Run Summary: + name lora_128_dim_512_H100 + fusion MLP (lightweight) + LoRA ON (rank=128) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_128_dim_512_H100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 512 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 300 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_128_dim_512_H100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 12.7/188.5 GiB (6.7%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.7s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 13.1/188.5 GiB (7.0%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (59.0s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 14.7/188.5 GiB (7.8%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 9.4M params +Control: 0.0M params +Total trainable: 9.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=128, alpha=128.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 39.32M + +Parameter Statistics: + HDC²A Adapter: total=9.4M trainable=9.4M + ControlNet (frozen): total=4172.8M LoRA trainable=39.32M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 9.4M + LoRA 39.32M = 48.77M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~4800 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0347, std=0.5039 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.350869 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.71 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.035889 + depth_encoder.conv_stem.6.weight: 0.032471 + semantic_encoder.conv_stem.3.weight: 0.022705 + depth_encoder.conv_stem.3.weight: 0.020752 + W_s.weight: 0.019043 + Test result: PASSED +[MEM @ after test] RAM: 15.2/188.5 GiB (8.1%) | VRAM: 38.0/93.1 GiB (40.8%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 672.097 GFLOPs transformer: 273441.526 GFLOPs total: 274113.623 GFLOPs + per-sample forward FLOPs: 274113.623 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 300 epochs × 66 steps = 19800 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 1980, 3960, 5940, 7920, 9900, 11879, 13859, 15839, 17819, 19799] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/299 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/66] loss=1.009864 avg=0.895808 VRAM=38.7GiB | 0.1% done | ETA(epoch): 6744s + [Epoch 0][20/66] loss=0.972565 avg=0.910056 VRAM=38.6GiB | 0.1% done | ETA(epoch): 3213s + [Epoch 0][30/66] loss=0.984125 avg=0.915839 VRAM=38.7GiB | 0.2% done | ETA(epoch): 1907s + [Epoch 0][40/66] loss=1.036940 avg=0.920781 VRAM=38.6GiB | 0.2% done | ETA(epoch): 1158s + [Epoch 0][50/66] loss=1.007875 avg=0.923174 VRAM=38.7GiB | 0.3% done | ETA(epoch): 632s + [Epoch 0][60/66] loss=0.974190 avg=0.914450 VRAM=38.6GiB | 0.3% done | ETA(epoch): 217s + Train loss: 0.913676 (2286.2s) ETA: 11415min + Val loss: 0.983590 [t_0.0-0.2=1.0969 t_0.2-0.4=1.0573 t_0.4-0.6=0.8935 t_0.6-0.8=0.7610 t_0.8-1.0=1.2208] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 1/299 (0% done) --- + [Epoch 1][10/66] loss=0.869710 avg=0.910784 VRAM=38.7GiB | 0.4% done | ETA(epoch): 1084s + [Epoch 1][20/66] loss=0.939176 avg=0.905765 VRAM=38.6GiB | 0.4% done | ETA(epoch): 887s + [Epoch 1][30/66] loss=0.977605 avg=0.907435 VRAM=38.7GiB | 0.5% done | ETA(epoch): 694s + [Epoch 1][40/66] loss=0.936588 avg=0.905888 VRAM=38.6GiB | 0.5% done | ETA(epoch): 501s + [Epoch 1][50/66] loss=0.912928 avg=0.907713 VRAM=38.7GiB | 0.6% done | ETA(epoch): 308s + [Epoch 1][60/66] loss=0.872662 avg=0.908097 VRAM=38.6GiB | 0.6% done | ETA(epoch): 116s + Train loss: 0.906855 (1271.3s) ETA: 9079min + Val loss: 0.971690 [t_0.0-0.2=1.0884 t_0.2-0.4=1.0387 t_0.4-0.6=0.8795 t_0.6-0.8=0.7632 t_0.8-1.0=1.0611] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 2/299 (1% done) --- + [Epoch 2][10/66] loss=0.944325 avg=0.930917 VRAM=38.7GiB | 0.7% done | ETA(epoch): 1078s + [Epoch 2][20/66] loss=0.795080 avg=0.934827 VRAM=38.6GiB | 0.8% done | ETA(epoch): 885s + [Epoch 2][30/66] loss=0.938770 avg=0.932447 VRAM=38.7GiB | 0.8% done | ETA(epoch): 693s + [Epoch 2][40/66] loss=0.865406 avg=0.926226 VRAM=38.6GiB | 0.9% done | ETA(epoch): 500s + [Epoch 2][50/66] loss=0.964584 avg=0.929361 VRAM=38.7GiB | 0.9% done | ETA(epoch): 309s + [Epoch 2][60/66] loss=0.915467 avg=0.924398 VRAM=38.6GiB | 1.0% done | ETA(epoch): 116s + Train loss: 0.921698 (1275.3s) ETA: 8293min + Val loss: 0.955505 [t_0.0-0.2=1.0840 t_0.2-0.4=1.0176 t_0.4-0.6=0.9122 t_0.6-0.8=0.7626 t_0.8-1.0=0.9271] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0002 (BEST) +[MEM @ epoch 2 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 3/299 (1% done) --- + [Epoch 3][10/66] loss=0.975400 avg=0.917040 VRAM=38.7GiB | 1.1% done | ETA(epoch): 1077s + [Epoch 3][20/66] loss=0.810875 avg=0.903594 VRAM=38.6GiB | 1.1% done | ETA(epoch): 886s + [Epoch 3][30/66] loss=0.903459 avg=0.898993 VRAM=38.7GiB | 1.2% done | ETA(epoch): 693s + [Epoch 3][40/66] loss=0.933859 avg=0.905058 VRAM=38.6GiB | 1.2% done | ETA(epoch): 501s + [Epoch 3][50/66] loss=0.958282 avg=0.903798 VRAM=38.7GiB | 1.3% done | ETA(epoch): 308s + [Epoch 3][60/66] loss=0.871687 avg=0.903271 VRAM=38.6GiB | 1.3% done | ETA(epoch): 116s + Train loss: 0.903173 (1271.0s) ETA: 7883min + Val loss: 0.983942 [t_0.0-0.2=1.0964 t_0.2-0.4=1.0031 t_0.4-0.6=0.8597 t_0.6-0.8=0.7730 t_0.8-1.0=1.0041] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 4/299 (1% done) --- + [Epoch 4][10/66] loss=0.892058 avg=0.880796 VRAM=38.7GiB | 1.4% done | ETA(epoch): 1077s + [Epoch 4][20/66] loss=0.866788 avg=0.890880 VRAM=38.6GiB | 1.4% done | ETA(epoch): 885s + [Epoch 4][30/66] loss=0.859495 avg=0.887039 VRAM=38.7GiB | 1.5% done | ETA(epoch): 693s + [Epoch 4][40/66] loss=0.811758 avg=0.882804 VRAM=38.6GiB | 1.5% done | ETA(epoch): 501s + [Epoch 4][50/66] loss=0.952058 avg=0.887590 VRAM=38.7GiB | 1.6% done | ETA(epoch): 308s + [Epoch 4][60/66] loss=0.938833 avg=0.887905 VRAM=38.6GiB | 1.6% done | ETA(epoch): 116s + Train loss: 0.891030 (1270.7s) ETA: 7628min + Val loss: 0.901731 [t_0.0-0.2=1.0986 t_0.2-0.4=1.0158 t_0.4-0.6=0.7993 t_0.6-0.8=0.7423 t_0.8-1.0=0.9023] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0004 (BEST) + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 5/299 (2% done) --- + [Epoch 5][10/66] loss=0.936078 avg=0.867243 VRAM=38.7GiB | 1.7% done | ETA(epoch): 1077s + [Epoch 5][20/66] loss=0.836313 avg=0.883254 VRAM=38.6GiB | 1.8% done | ETA(epoch): 885s + [Epoch 5][30/66] loss=0.906815 avg=0.886041 VRAM=38.7GiB | 1.8% done | ETA(epoch): 695s + [Epoch 5][40/66] loss=0.975250 avg=0.891003 VRAM=38.6GiB | 1.9% done | ETA(epoch): 502s + [Epoch 5][50/66] loss=0.932049 avg=0.889707 VRAM=38.7GiB | 1.9% done | ETA(epoch): 309s + [Epoch 5][60/66] loss=0.933814 avg=0.895149 VRAM=38.6GiB | 2.0% done | ETA(epoch): 116s + Train loss: 0.893848 (1273.0s) ETA: 7453min + Val loss: 0.893321 [t_0.0-0.2=1.0984 t_0.2-0.4=1.0162 t_0.4-0.6=0.8303 t_0.6-0.8=0.7492 t_0.8-1.0=0.8037] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0005 (BEST) + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 6/299 (2% done) --- + [Epoch 6][10/66] loss=0.889533 avg=0.887488 VRAM=38.7GiB | 2.1% done | ETA(epoch): 1077s + [Epoch 6][20/66] loss=0.929415 avg=0.893249 VRAM=38.6GiB | 2.1% done | ETA(epoch): 886s + [Epoch 6][30/66] loss=0.837748 avg=0.880907 VRAM=38.7GiB | 2.2% done | ETA(epoch): 694s + [Epoch 6][40/66] loss=0.842090 avg=0.885214 VRAM=38.6GiB | 2.2% done | ETA(epoch): 501s + [Epoch 6][50/66] loss=0.996590 avg=0.886290 VRAM=38.7GiB | 2.3% done | ETA(epoch): 308s + [Epoch 6][60/66] loss=0.994836 avg=0.890356 VRAM=38.6GiB | 2.3% done | ETA(epoch): 116s + Train loss: 0.887013 (1271.8s) ETA: 7320min + Val loss: 0.879437 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9793 t_0.4-0.6=0.8294 t_0.6-0.8=0.7559 t_0.8-1.0=0.7854] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0006 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 7/299 (2% done) --- + [Epoch 7][10/66] loss=1.007792 avg=0.894619 VRAM=38.7GiB | 2.4% done | ETA(epoch): 1088s + [Epoch 7][20/66] loss=0.847033 avg=0.894303 VRAM=38.6GiB | 2.4% done | ETA(epoch): 890s + [Epoch 7][30/66] loss=0.800524 avg=0.897100 VRAM=38.7GiB | 2.5% done | ETA(epoch): 695s + [Epoch 7][40/66] loss=0.850841 avg=0.899684 VRAM=38.6GiB | 2.5% done | ETA(epoch): 502s + [Epoch 7][50/66] loss=0.735125 avg=0.892565 VRAM=38.7GiB | 2.6% done | ETA(epoch): 309s + [Epoch 7][60/66] loss=0.941802 avg=0.891199 VRAM=38.6GiB | 2.6% done | ETA(epoch): 116s + Train loss: 0.892666 (1273.0s) ETA: 7216min + Val loss: 0.891807 [t_0.0-0.2=1.0549 t_0.2-0.4=1.0348 t_0.4-0.6=0.8784 t_0.6-0.8=0.7459 t_0.8-1.0=0.7602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 8/299 (3% done) --- + [Epoch 8][10/66] loss=0.984413 avg=0.899358 VRAM=38.7GiB | 2.7% done | ETA(epoch): 1077s + [Epoch 8][20/66] loss=0.967338 avg=0.889266 VRAM=38.6GiB | 2.8% done | ETA(epoch): 885s + [Epoch 8][30/66] loss=0.831910 avg=0.887982 VRAM=38.7GiB | 2.8% done | ETA(epoch): 693s + [Epoch 8][40/66] loss=0.990462 avg=0.898972 VRAM=38.6GiB | 2.9% done | ETA(epoch): 501s + [Epoch 8][50/66] loss=0.855176 avg=0.895848 VRAM=38.7GiB | 2.9% done | ETA(epoch): 308s + [Epoch 8][60/66] loss=0.874795 avg=0.894705 VRAM=38.6GiB | 3.0% done | ETA(epoch): 116s + Train loss: 0.890402 (1271.3s) ETA: 7130min + Val loss: 0.903192 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0123 t_0.4-0.6=0.8449 t_0.6-0.8=0.7611 t_0.8-1.0=0.7689] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 9/299 (3% done) --- + [Epoch 9][10/66] loss=0.807423 avg=0.866653 VRAM=38.7GiB | 3.1% done | ETA(epoch): 1078s + [Epoch 9][20/66] loss=0.791603 avg=0.873449 VRAM=38.6GiB | 3.1% done | ETA(epoch): 886s + [Epoch 9][30/66] loss=0.907419 avg=0.883651 VRAM=38.7GiB | 3.2% done | ETA(epoch): 693s + [Epoch 9][40/66] loss=0.913629 avg=0.893140 VRAM=38.6GiB | 3.2% done | ETA(epoch): 501s + [Epoch 9][50/66] loss=0.794450 avg=0.884031 VRAM=38.7GiB | 3.3% done | ETA(epoch): 308s + [Epoch 9][60/66] loss=0.941124 avg=0.885932 VRAM=38.6GiB | 3.3% done | ETA(epoch): 116s + Train loss: 0.885059 (1271.4s) ETA: 7055min + Val loss: 0.901717 [t_0.0-0.2=1.0626 t_0.2-0.4=0.9905 t_0.4-0.6=0.8657 t_0.6-0.8=0.7575 t_0.8-1.0=0.7616] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0009 +[MEM @ epoch 9 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 10/299 (3% done) --- + [Epoch 10][10/66] loss=0.818717 avg=0.879775 VRAM=38.7GiB | 3.4% done | ETA(epoch): 1080s + [Epoch 10][20/66] loss=0.778497 avg=0.873199 VRAM=38.6GiB | 3.4% done | ETA(epoch): 887s + [Epoch 10][30/66] loss=0.810585 avg=0.874639 VRAM=38.7GiB | 3.5% done | ETA(epoch): 694s + [Epoch 10][40/66] loss=0.788622 avg=0.878395 VRAM=38.6GiB | 3.5% done | ETA(epoch): 501s + [Epoch 10][50/66] loss=0.822612 avg=0.881643 VRAM=38.7GiB | 3.6% done | ETA(epoch): 308s + [Epoch 10][60/66] loss=0.838074 avg=0.879088 VRAM=38.6GiB | 3.6% done | ETA(epoch): 116s + Train loss: 0.875833 (1271.4s) ETA: 6990min + Val loss: 0.877179 [t_0.0-0.2=1.0900 t_0.2-0.4=1.0080 t_0.4-0.6=0.8474 t_0.6-0.8=0.7427 t_0.8-1.0=0.7402] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0010 (BEST) + Deleted old checkpoint: checkpoint_epoch_0006 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 11/299 (4% done) --- + [Epoch 11][10/66] loss=0.856650 avg=0.909726 VRAM=38.7GiB | 3.7% done | ETA(epoch): 1080s + [Epoch 11][20/66] loss=0.949534 avg=0.902644 VRAM=38.6GiB | 3.8% done | ETA(epoch): 886s + [Epoch 11][30/66] loss=0.886562 avg=0.892980 VRAM=38.7GiB | 3.8% done | ETA(epoch): 693s + [Epoch 11][40/66] loss=0.827948 avg=0.889444 VRAM=38.6GiB | 3.9% done | ETA(epoch): 501s + [Epoch 11][50/66] loss=0.851329 avg=0.884748 VRAM=38.7GiB | 3.9% done | ETA(epoch): 308s + [Epoch 11][60/66] loss=0.886812 avg=0.882928 VRAM=38.6GiB | 4.0% done | ETA(epoch): 116s + Train loss: 0.886759 (1270.7s) ETA: 6932min + Val loss: 0.853299 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9767 t_0.4-0.6=0.8087 t_0.6-0.8=0.7547 t_0.8-1.0=0.7566] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0011 (BEST) + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 12/299 (4% done) --- + [Epoch 12][10/66] loss=0.923366 avg=0.875145 VRAM=38.7GiB | 4.1% done | ETA(epoch): 1077s + [Epoch 12][20/66] loss=0.937119 avg=0.875271 VRAM=38.6GiB | 4.1% done | ETA(epoch): 885s + [Epoch 12][30/66] loss=0.880023 avg=0.868456 VRAM=38.7GiB | 4.2% done | ETA(epoch): 697s + [Epoch 12][40/66] loss=0.829426 avg=0.866370 VRAM=38.6GiB | 4.2% done | ETA(epoch): 502s + [Epoch 12][50/66] loss=0.996343 avg=0.866485 VRAM=38.7GiB | 4.3% done | ETA(epoch): 309s + [Epoch 12][60/66] loss=0.828035 avg=0.865732 VRAM=38.6GiB | 4.3% done | ETA(epoch): 116s + Train loss: 0.864647 (1272.5s) ETA: 6880min + Val loss: 0.891816 [t_0.0-0.2=1.0672 t_0.2-0.4=1.0211 t_0.4-0.6=0.8477 t_0.6-0.8=0.7397 t_0.8-1.0=0.7519] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 13/299 (4% done) --- + [Epoch 13][10/66] loss=0.776739 avg=0.879563 VRAM=38.7GiB | 4.4% done | ETA(epoch): 1079s + [Epoch 13][20/66] loss=0.846821 avg=0.858908 VRAM=38.6GiB | 4.4% done | ETA(epoch): 890s + [Epoch 13][30/66] loss=0.909563 avg=0.876808 VRAM=38.7GiB | 4.5% done | ETA(epoch): 696s + [Epoch 13][40/66] loss=0.869317 avg=0.875363 VRAM=38.6GiB | 4.5% done | ETA(epoch): 501s + [Epoch 13][50/66] loss=0.817304 avg=0.876206 VRAM=38.7GiB | 4.6% done | ETA(epoch): 308s + [Epoch 13][60/66] loss=0.770912 avg=0.869330 VRAM=38.6GiB | 4.6% done | ETA(epoch): 116s + Train loss: 0.873809 (1271.4s) ETA: 6831min + Val loss: 0.882470 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0076 t_0.4-0.6=0.8680 t_0.6-0.8=0.7190 t_0.8-1.0=0.7449] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 14/299 (5% done) --- + [Epoch 14][10/66] loss=0.916051 avg=0.877706 VRAM=38.7GiB | 4.7% done | ETA(epoch): 1076s + [Epoch 14][20/66] loss=0.943182 avg=0.892759 VRAM=38.6GiB | 4.8% done | ETA(epoch): 889s + [Epoch 14][30/66] loss=0.972372 avg=0.891430 VRAM=38.7GiB | 4.8% done | ETA(epoch): 694s + [Epoch 14][40/66] loss=0.946988 avg=0.885028 VRAM=38.6GiB | 4.9% done | ETA(epoch): 501s + [Epoch 14][50/66] loss=0.843846 avg=0.881852 VRAM=38.7GiB | 4.9% done | ETA(epoch): 308s + [Epoch 14][60/66] loss=0.864854 avg=0.885532 VRAM=38.6GiB | 5.0% done | ETA(epoch): 116s + Train loss: 0.882252 (1271.8s) ETA: 6786min + Val loss: 0.865201 [t_0.0-0.2=1.0773 t_0.2-0.4=0.9722 t_0.4-0.6=0.8676 t_0.6-0.8=0.7436 t_0.8-1.0=0.7267] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0014 +[MEM @ epoch 14 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 15/299 (5% done) --- + [Epoch 15][10/66] loss=0.932412 avg=0.899684 VRAM=38.7GiB | 5.1% done | ETA(epoch): 1096s + [Epoch 15][20/66] loss=0.760710 avg=0.896718 VRAM=38.6GiB | 5.1% done | ETA(epoch): 893s + [Epoch 15][30/66] loss=0.873878 avg=0.884504 VRAM=38.7GiB | 5.2% done | ETA(epoch): 696s + [Epoch 15][40/66] loss=1.010499 avg=0.887609 VRAM=38.6GiB | 5.2% done | ETA(epoch): 502s + [Epoch 15][50/66] loss=0.868148 avg=0.888226 VRAM=38.7GiB | 5.3% done | ETA(epoch): 309s + [Epoch 15][60/66] loss=0.978417 avg=0.891553 VRAM=38.6GiB | 5.3% done | ETA(epoch): 116s + Train loss: 0.887850 (1272.5s) ETA: 6745min + Val loss: 0.852138 [t_0.0-0.2=1.0659 t_0.2-0.4=0.9860 t_0.4-0.6=0.8791 t_0.6-0.8=0.7272 t_0.8-1.0=0.7195] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0015 (BEST) + Deleted old checkpoint: checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 16/299 (5% done) --- + [Epoch 16][10/66] loss=0.839584 avg=0.875239 VRAM=38.7GiB | 5.4% done | ETA(epoch): 1076s + [Epoch 16][20/66] loss=0.873915 avg=0.887109 VRAM=38.6GiB | 5.4% done | ETA(epoch): 884s + [Epoch 16][30/66] loss=0.832711 avg=0.886172 VRAM=38.7GiB | 5.5% done | ETA(epoch): 692s + [Epoch 16][40/66] loss=0.965951 avg=0.883102 VRAM=38.6GiB | 5.5% done | ETA(epoch): 500s + [Epoch 16][50/66] loss=0.807490 avg=0.879373 VRAM=38.7GiB | 5.6% done | ETA(epoch): 308s + [Epoch 16][60/66] loss=0.917980 avg=0.876048 VRAM=38.6GiB | 5.6% done | ETA(epoch): 115s + Train loss: 0.877614 (1269.4s) ETA: 6704min + Val loss: 0.902386 [t_0.0-0.2=1.0802 t_0.2-0.4=1.0254 t_0.4-0.6=0.8529 t_0.6-0.8=0.7411 t_0.8-1.0=0.7333] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 17/299 (6% done) --- + [Epoch 17][10/66] loss=0.724764 avg=0.862340 VRAM=38.7GiB | 5.7% done | ETA(epoch): 1076s + [Epoch 17][20/66] loss=0.791333 avg=0.861153 VRAM=38.6GiB | 5.8% done | ETA(epoch): 885s + [Epoch 17][30/66] loss=0.865173 avg=0.868261 VRAM=38.7GiB | 5.8% done | ETA(epoch): 692s + [Epoch 17][40/66] loss=0.930043 avg=0.879819 VRAM=38.6GiB | 5.9% done | ETA(epoch): 500s + [Epoch 17][50/66] loss=0.914103 avg=0.882226 VRAM=38.7GiB | 5.9% done | ETA(epoch): 308s + [Epoch 17][60/66] loss=0.889246 avg=0.886350 VRAM=38.6GiB | 6.0% done | ETA(epoch): 115s + Train loss: 0.882275 (1269.5s) ETA: 6666min + Val loss: 0.889376 [t_0.0-0.2=1.0805 t_0.2-0.4=1.0066 t_0.4-0.6=0.8804 t_0.6-0.8=0.7110 t_0.8-1.0=0.7094] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 18/299 (6% done) --- + [Epoch 18][10/66] loss=0.837197 avg=0.890424 VRAM=38.7GiB | 6.1% done | ETA(epoch): 1077s + [Epoch 18][20/66] loss=0.916753 avg=0.875331 VRAM=38.6GiB | 6.1% done | ETA(epoch): 885s + [Epoch 18][30/66] loss=0.752711 avg=0.878907 VRAM=38.7GiB | 6.2% done | ETA(epoch): 692s + [Epoch 18][40/66] loss=0.804433 avg=0.883962 VRAM=38.6GiB | 6.2% done | ETA(epoch): 500s + [Epoch 18][50/66] loss=0.869913 avg=0.882262 VRAM=38.7GiB | 6.3% done | ETA(epoch): 308s + [Epoch 18][60/66] loss=0.848073 avg=0.880678 VRAM=38.6GiB | 6.3% done | ETA(epoch): 115s + Train loss: 0.880677 (1269.7s) ETA: 6629min + Val loss: 0.899518 [t_0.0-0.2=1.0798 t_0.2-0.4=0.9775 t_0.4-0.6=0.8400 t_0.6-0.8=0.7323 t_0.8-1.0=0.7151] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0018 +[MEM @ epoch 18 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 19/299 (6% done) --- + [Epoch 19][10/66] loss=0.893114 avg=0.851207 VRAM=38.7GiB | 6.4% done | ETA(epoch): 1076s + [Epoch 19][20/66] loss=0.934958 avg=0.853537 VRAM=38.6GiB | 6.4% done | ETA(epoch): 884s + [Epoch 19][30/66] loss=0.855394 avg=0.863215 VRAM=38.7GiB | 6.5% done | ETA(epoch): 692s + [Epoch 19][40/66] loss=0.890733 avg=0.864930 VRAM=38.6GiB | 6.5% done | ETA(epoch): 500s + [Epoch 19][50/66] loss=0.901161 avg=0.870340 VRAM=38.7GiB | 6.6% done | ETA(epoch): 308s + [Epoch 19][60/66] loss=0.858472 avg=0.871953 VRAM=38.6GiB | 6.6% done | ETA(epoch): 115s + Train loss: 0.871194 (1268.4s) ETA: 6593min + Val loss: 0.850663 [t_0.0-0.2=1.0821 t_0.2-0.4=0.9925 t_0.4-0.6=0.8809 t_0.6-0.8=0.7029 t_0.8-1.0=0.7206] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0019 (BEST) + Deleted old checkpoint: checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 20/299 (7% done) --- + [Epoch 20][10/66] loss=0.903568 avg=0.821765 VRAM=38.7GiB | 6.7% done | ETA(epoch): 1076s + [Epoch 20][20/66] loss=0.870017 avg=0.851726 VRAM=38.6GiB | 6.8% done | ETA(epoch): 884s + [Epoch 20][30/66] loss=0.818313 avg=0.848669 VRAM=38.7GiB | 6.8% done | ETA(epoch): 692s + [Epoch 20][40/66] loss=0.965108 avg=0.854213 VRAM=38.6GiB | 6.9% done | ETA(epoch): 500s + [Epoch 20][50/66] loss=0.777283 avg=0.861759 VRAM=38.7GiB | 6.9% done | ETA(epoch): 308s + [Epoch 20][60/66] loss=0.997348 avg=0.867813 VRAM=38.6GiB | 7.0% done | ETA(epoch): 115s + Train loss: 0.866928 (1269.6s) ETA: 6559min + Val loss: 0.859882 [t_0.0-0.2=1.0739 t_0.2-0.4=1.0086 t_0.4-0.6=0.8515 t_0.6-0.8=0.7393 t_0.8-1.0=0.7153] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 21/299 (7% done) --- + [Epoch 21][10/66] loss=0.900239 avg=0.838705 VRAM=38.7GiB | 7.1% done | ETA(epoch): 1078s + [Epoch 21][20/66] loss=0.826738 avg=0.847394 VRAM=38.6GiB | 7.1% done | ETA(epoch): 886s + [Epoch 21][30/66] loss=0.850350 avg=0.862556 VRAM=38.7GiB | 7.2% done | ETA(epoch): 693s + [Epoch 21][40/66] loss=0.989240 avg=0.866175 VRAM=38.6GiB | 7.2% done | ETA(epoch): 500s + [Epoch 21][50/66] loss=0.893815 avg=0.869995 VRAM=38.7GiB | 7.3% done | ETA(epoch): 308s + [Epoch 21][60/66] loss=0.876490 avg=0.871838 VRAM=38.6GiB | 7.3% done | ETA(epoch): 115s + Train loss: 0.871438 (1270.2s) ETA: 6526min + Val loss: 0.842528 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9247 t_0.4-0.6=0.8509 t_0.6-0.8=0.7430 t_0.8-1.0=0.7110] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0021 (BEST) + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 22/299 (7% done) --- + [Epoch 22][10/66] loss=0.710632 avg=0.881590 VRAM=38.7GiB | 7.4% done | ETA(epoch): 1075s + [Epoch 22][20/66] loss=0.930723 avg=0.867518 VRAM=38.6GiB | 7.4% done | ETA(epoch): 884s + [Epoch 22][30/66] loss=0.823472 avg=0.868787 VRAM=38.7GiB | 7.5% done | ETA(epoch): 692s + [Epoch 22][40/66] loss=0.947556 avg=0.869391 VRAM=38.6GiB | 7.5% done | ETA(epoch): 500s + [Epoch 22][50/66] loss=0.849307 avg=0.865973 VRAM=38.7GiB | 7.6% done | ETA(epoch): 308s + [Epoch 22][60/66] loss=0.907831 avg=0.868622 VRAM=38.6GiB | 7.6% done | ETA(epoch): 115s + Train loss: 0.868764 (1269.5s) ETA: 6494min + Val loss: 0.871167 [t_0.0-0.2=1.0807 t_0.2-0.4=0.9972 t_0.4-0.6=0.8589 t_0.6-0.8=0.7333 t_0.8-1.0=0.7172] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 23/299 (8% done) --- + [Epoch 23][10/66] loss=0.947867 avg=0.891939 VRAM=38.7GiB | 7.7% done | ETA(epoch): 1076s + [Epoch 23][20/66] loss=0.854231 avg=0.890585 VRAM=38.6GiB | 7.8% done | ETA(epoch): 884s + [Epoch 23][30/66] loss=0.974306 avg=0.892008 VRAM=38.7GiB | 7.8% done | ETA(epoch): 692s + [Epoch 23][40/66] loss=0.793579 avg=0.889364 VRAM=38.6GiB | 7.9% done | ETA(epoch): 500s + [Epoch 23][50/66] loss=0.941279 avg=0.890880 VRAM=38.7GiB | 7.9% done | ETA(epoch): 308s + [Epoch 23][60/66] loss=0.933099 avg=0.887476 VRAM=38.6GiB | 8.0% done | ETA(epoch): 115s + Train loss: 0.886359 (1268.8s) ETA: 6463min + Val loss: 0.868818 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0001 t_0.4-0.6=0.8027 t_0.6-0.8=0.7162 t_0.8-1.0=0.7208] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 24/299 (8% done) --- + [Epoch 24][10/66] loss=0.845080 avg=0.883019 VRAM=38.7GiB | 8.1% done | ETA(epoch): 1077s + [Epoch 24][20/66] loss=0.824124 avg=0.871702 VRAM=38.6GiB | 8.1% done | ETA(epoch): 885s + [Epoch 24][30/66] loss=0.988884 avg=0.880992 VRAM=38.7GiB | 8.2% done | ETA(epoch): 692s + [Epoch 24][40/66] loss=0.852048 avg=0.878211 VRAM=38.6GiB | 8.2% done | ETA(epoch): 500s + [Epoch 24][50/66] loss=0.970829 avg=0.882704 VRAM=38.7GiB | 8.3% done | ETA(epoch): 308s + [Epoch 24][60/66] loss=0.883840 avg=0.881208 VRAM=38.6GiB | 8.3% done | ETA(epoch): 116s + Train loss: 0.881548 (1271.6s) ETA: 6433min + Val loss: 0.877819 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9919 t_0.4-0.6=0.8769 t_0.6-0.8=0.7151 t_0.8-1.0=0.6916] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0024 +[MEM @ epoch 24 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 25/299 (8% done) --- + [Epoch 25][10/66] loss=0.972317 avg=0.863165 VRAM=38.7GiB | 8.4% done | ETA(epoch): 1076s + [Epoch 25][20/66] loss=0.796682 avg=0.872708 VRAM=38.6GiB | 8.4% done | ETA(epoch): 884s + [Epoch 25][30/66] loss=0.891493 avg=0.878759 VRAM=38.7GiB | 8.5% done | ETA(epoch): 693s + [Epoch 25][40/66] loss=0.920619 avg=0.877171 VRAM=38.6GiB | 8.5% done | ETA(epoch): 500s + [Epoch 25][50/66] loss=0.843226 avg=0.880913 VRAM=38.7GiB | 8.6% done | ETA(epoch): 308s + [Epoch 25][60/66] loss=0.851944 avg=0.878137 VRAM=38.6GiB | 8.6% done | ETA(epoch): 116s + Train loss: 0.879496 (1271.6s) ETA: 6403min + Val loss: 0.894972 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0079 t_0.4-0.6=0.8575 t_0.6-0.8=0.7407 t_0.8-1.0=0.7291] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 26/299 (9% done) --- + [Epoch 26][10/66] loss=0.737092 avg=0.882164 VRAM=38.7GiB | 8.7% done | ETA(epoch): 1075s + [Epoch 26][20/66] loss=0.910713 avg=0.887906 VRAM=38.6GiB | 8.8% done | ETA(epoch): 884s + [Epoch 26][30/66] loss=0.957518 avg=0.886812 VRAM=38.7GiB | 8.8% done | ETA(epoch): 692s + [Epoch 26][40/66] loss=0.884717 avg=0.886812 VRAM=38.6GiB | 8.9% done | ETA(epoch): 500s + [Epoch 26][50/66] loss=0.964137 avg=0.886429 VRAM=38.7GiB | 8.9% done | ETA(epoch): 308s + [Epoch 26][60/66] loss=0.919806 avg=0.880929 VRAM=38.6GiB | 9.0% done | ETA(epoch): 115s + Train loss: 0.879471 (1269.7s) ETA: 6373min + Val loss: 0.844671 [t_0.0-0.2=1.0756 t_0.2-0.4=1.0203 t_0.4-0.6=0.8570 t_0.6-0.8=0.7307 t_0.8-1.0=0.7024] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 27/299 (9% done) --- + [Epoch 27][10/66] loss=0.865480 avg=0.867293 VRAM=38.7GiB | 9.1% done | ETA(epoch): 1076s + [Epoch 27][20/66] loss=0.855582 avg=0.864583 VRAM=38.6GiB | 9.1% done | ETA(epoch): 884s + [Epoch 27][30/66] loss=0.944595 avg=0.876546 VRAM=38.7GiB | 9.2% done | ETA(epoch): 692s + [Epoch 27][40/66] loss=0.933015 avg=0.875681 VRAM=38.6GiB | 9.2% done | ETA(epoch): 500s + [Epoch 27][50/66] loss=0.892853 avg=0.876785 VRAM=38.7GiB | 9.3% done | ETA(epoch): 308s + [Epoch 27][60/66] loss=0.843397 avg=0.870034 VRAM=38.6GiB | 9.3% done | ETA(epoch): 115s + Train loss: 0.865974 (1269.7s) ETA: 6344min + Val loss: 0.883712 [t_0.0-0.2=1.0734 t_0.2-0.4=1.0067 t_0.4-0.6=0.8136 t_0.6-0.8=0.7478 t_0.8-1.0=0.6930] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 28/299 (9% done) --- + [Epoch 28][10/66] loss=0.907131 avg=0.853670 VRAM=38.7GiB | 9.4% done | ETA(epoch): 1078s + [Epoch 28][20/66] loss=0.865141 avg=0.868027 VRAM=38.6GiB | 9.4% done | ETA(epoch): 885s + [Epoch 28][30/66] loss=0.957054 avg=0.856159 VRAM=38.7GiB | 9.5% done | ETA(epoch): 693s + [Epoch 28][40/66] loss=0.933171 avg=0.861641 VRAM=38.6GiB | 9.5% done | ETA(epoch): 501s + [Epoch 28][50/66] loss=0.953561 avg=0.862482 VRAM=38.7GiB | 9.6% done | ETA(epoch): 308s + [Epoch 28][60/66] loss=0.869572 avg=0.866096 VRAM=38.6GiB | 9.6% done | ETA(epoch): 116s + Train loss: 0.866992 (1270.5s) ETA: 6315min + Val loss: 0.849337 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9866 t_0.4-0.6=0.9001 t_0.6-0.8=0.7346 t_0.8-1.0=0.6812] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 29/299 (10% done) --- + [Epoch 29][10/66] loss=0.788389 avg=0.869621 VRAM=38.7GiB | 9.7% done | ETA(epoch): 1076s + [Epoch 29][20/66] loss=0.752659 avg=0.883994 VRAM=38.6GiB | 9.8% done | ETA(epoch): 885s + [Epoch 29][30/66] loss=0.761239 avg=0.871636 VRAM=38.7GiB | 9.8% done | ETA(epoch): 693s + [Epoch 29][40/66] loss=0.980570 avg=0.871669 VRAM=38.6GiB | 9.9% done | ETA(epoch): 500s + [Epoch 29][50/66] loss=0.864238 avg=0.864235 VRAM=38.7GiB | 9.9% done | ETA(epoch): 308s + [Epoch 29][60/66] loss=0.857725 avg=0.863335 VRAM=38.6GiB | 10.0% done | ETA(epoch): 115s + Train loss: 0.861784 (1269.1s) ETA: 6287min + Val loss: 0.867931 [t_0.0-0.2=1.0709 t_0.2-0.4=1.0125 t_0.4-0.6=0.8217 t_0.6-0.8=0.7261 t_0.8-1.0=0.6775] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 30/299 (10% done) --- + [MilestoneVis] train_0 step 1980 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 1980 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 1980 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 1980 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 1980 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 1980 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 1980 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 1980 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 1980 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 1980 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 30][10/66] loss=0.884425 avg=0.905279 VRAM=38.7GiB | 10.1% done | ETA(epoch): 6807s + [Epoch 30][20/66] loss=0.969780 avg=0.906040 VRAM=38.6GiB | 10.1% done | ETA(epoch): 3244s + [Epoch 30][30/66] loss=0.798725 avg=0.892955 VRAM=38.7GiB | 10.2% done | ETA(epoch): 1923s + [Epoch 30][40/66] loss=0.889121 avg=0.878046 VRAM=38.6GiB | 10.2% done | ETA(epoch): 1167s + [Epoch 30][50/66] loss=0.806799 avg=0.880574 VRAM=38.7GiB | 10.3% done | ETA(epoch): 636s + [Epoch 30][60/66] loss=0.783486 avg=0.870495 VRAM=38.6GiB | 10.3% done | ETA(epoch): 218s + Train loss: 0.874437 (2295.6s) ETA: 6407min + Val loss: 0.860710 [t_0.0-0.2=1.0702 t_0.2-0.4=1.0221 t_0.4-0.6=0.8632 t_0.6-0.8=0.7154 t_0.8-1.0=0.6938] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 31/299 (10% done) --- + [Epoch 31][10/66] loss=0.760449 avg=0.868203 VRAM=38.7GiB | 10.4% done | ETA(epoch): 1078s + [Epoch 31][20/66] loss=0.833617 avg=0.882905 VRAM=38.6GiB | 10.4% done | ETA(epoch): 885s + [Epoch 31][30/66] loss=0.878376 avg=0.886400 VRAM=38.7GiB | 10.5% done | ETA(epoch): 692s + [Epoch 31][40/66] loss=0.903940 avg=0.880291 VRAM=38.6GiB | 10.5% done | ETA(epoch): 500s + [Epoch 31][50/66] loss=0.845762 avg=0.885785 VRAM=38.7GiB | 10.6% done | ETA(epoch): 308s + [Epoch 31][60/66] loss=0.916436 avg=0.883611 VRAM=38.6GiB | 10.6% done | ETA(epoch): 115s + Train loss: 0.879251 (1269.7s) ETA: 6375min + Val loss: 0.847415 [t_0.0-0.2=1.0850 t_0.2-0.4=1.0236 t_0.4-0.6=0.8311 t_0.6-0.8=0.7203 t_0.8-1.0=0.6931] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 32/299 (11% done) --- + [Epoch 32][10/66] loss=0.912626 avg=0.891068 VRAM=38.7GiB | 10.7% done | ETA(epoch): 1077s + [Epoch 32][20/66] loss=0.909465 avg=0.881951 VRAM=38.6GiB | 10.8% done | ETA(epoch): 885s + [Epoch 32][30/66] loss=0.885473 avg=0.884474 VRAM=38.7GiB | 10.8% done | ETA(epoch): 692s + [Epoch 32][40/66] loss=0.766945 avg=0.881385 VRAM=38.6GiB | 10.9% done | ETA(epoch): 500s + [Epoch 32][50/66] loss=1.025402 avg=0.887238 VRAM=38.7GiB | 10.9% done | ETA(epoch): 308s + [Epoch 32][60/66] loss=0.990121 avg=0.883414 VRAM=38.6GiB | 11.0% done | ETA(epoch): 115s + Train loss: 0.885572 (1269.5s) ETA: 6342min + Val loss: 0.838042 [t_0.0-0.2=1.0627 t_0.2-0.4=0.9968 t_0.4-0.6=0.8543 t_0.6-0.8=0.7339 t_0.8-1.0=0.6929] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0032 (BEST) + Deleted old checkpoint: checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 19.5/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 33/299 (11% done) --- + [Epoch 33][10/66] loss=0.888611 avg=0.884155 VRAM=38.7GiB | 11.1% done | ETA(epoch): 1076s + [Epoch 33][20/66] loss=0.932503 avg=0.882458 VRAM=38.6GiB | 11.1% done | ETA(epoch): 884s + [Epoch 33][30/66] loss=0.905205 avg=0.883351 VRAM=38.7GiB | 11.2% done | ETA(epoch): 692s + [Epoch 33][40/66] loss=0.912111 avg=0.881587 VRAM=38.6GiB | 11.2% done | ETA(epoch): 500s + [Epoch 33][50/66] loss=0.941059 avg=0.878185 VRAM=38.7GiB | 11.3% done | ETA(epoch): 308s + [Epoch 33][60/66] loss=0.854011 avg=0.874718 VRAM=38.6GiB | 11.3% done | ETA(epoch): 115s + Train loss: 0.871692 (1269.7s) ETA: 6311min + Val loss: 0.863222 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0126 t_0.4-0.6=0.8924 t_0.6-0.8=0.7372 t_0.8-1.0=0.6775] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 34/299 (11% done) --- + [Epoch 34][10/66] loss=0.916917 avg=0.883650 VRAM=38.7GiB | 11.4% done | ETA(epoch): 1077s + [Epoch 34][20/66] loss=0.896459 avg=0.887385 VRAM=38.6GiB | 11.4% done | ETA(epoch): 886s + [Epoch 34][30/66] loss=0.802722 avg=0.883946 VRAM=38.7GiB | 11.5% done | ETA(epoch): 693s + [Epoch 34][40/66] loss=0.880147 avg=0.872979 VRAM=38.6GiB | 11.5% done | ETA(epoch): 500s + [Epoch 34][50/66] loss=0.854509 avg=0.875702 VRAM=38.7GiB | 11.6% done | ETA(epoch): 308s + [Epoch 34][60/66] loss=0.834640 avg=0.876835 VRAM=38.6GiB | 11.6% done | ETA(epoch): 116s + Train loss: 0.875124 (1271.0s) ETA: 6280min + Val loss: 0.840910 [t_0.0-0.2=1.0987 t_0.2-0.4=0.9910 t_0.4-0.6=0.8426 t_0.6-0.8=0.7163 t_0.8-1.0=0.6745] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 35/299 (12% done) --- + [Epoch 35][10/66] loss=0.813171 avg=0.882032 VRAM=38.7GiB | 11.7% done | ETA(epoch): 1077s + [Epoch 35][20/66] loss=0.820703 avg=0.849764 VRAM=38.6GiB | 11.8% done | ETA(epoch): 886s + [Epoch 35][30/66] loss=0.831041 avg=0.863028 VRAM=38.7GiB | 11.8% done | ETA(epoch): 693s + [Epoch 35][40/66] loss=0.972508 avg=0.867131 VRAM=38.6GiB | 11.9% done | ETA(epoch): 502s + [Epoch 35][50/66] loss=0.794583 avg=0.872150 VRAM=38.7GiB | 11.9% done | ETA(epoch): 309s + [Epoch 35][60/66] loss=0.808102 avg=0.869557 VRAM=38.6GiB | 12.0% done | ETA(epoch): 116s + Train loss: 0.867468 (1272.9s) ETA: 6249min + Val loss: 0.874006 [t_0.0-0.2=1.0590 t_0.2-0.4=0.9975 t_0.4-0.6=0.8956 t_0.6-0.8=0.7105 t_0.8-1.0=0.6841] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0035 +[MEM @ epoch 35 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 36/299 (12% done) --- + [Epoch 36][10/66] loss=0.775997 avg=0.858814 VRAM=38.7GiB | 12.1% done | ETA(epoch): 1078s + [Epoch 36][20/66] loss=0.877830 avg=0.859600 VRAM=38.6GiB | 12.1% done | ETA(epoch): 885s + [Epoch 36][30/66] loss=0.969794 avg=0.875188 VRAM=38.7GiB | 12.2% done | ETA(epoch): 693s + [Epoch 36][40/66] loss=0.783953 avg=0.866214 VRAM=38.6GiB | 12.2% done | ETA(epoch): 501s + [Epoch 36][50/66] loss=0.929659 avg=0.867773 VRAM=38.7GiB | 12.3% done | ETA(epoch): 308s + [Epoch 36][60/66] loss=0.911507 avg=0.864055 VRAM=38.6GiB | 12.3% done | ETA(epoch): 116s + Train loss: 0.865301 (1271.2s) ETA: 6219min + Val loss: 0.851005 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0009 t_0.4-0.6=0.8597 t_0.6-0.8=0.7100 t_0.8-1.0=0.6862] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 37/299 (12% done) --- + [Epoch 37][10/66] loss=0.974746 avg=0.838863 VRAM=38.7GiB | 12.4% done | ETA(epoch): 1076s + [Epoch 37][20/66] loss=0.730672 avg=0.859161 VRAM=38.6GiB | 12.4% done | ETA(epoch): 884s + [Epoch 37][30/66] loss=0.862386 avg=0.855851 VRAM=38.7GiB | 12.5% done | ETA(epoch): 692s + [Epoch 37][40/66] loss=0.766706 avg=0.851452 VRAM=38.6GiB | 12.5% done | ETA(epoch): 500s + [Epoch 37][50/66] loss=0.847124 avg=0.846674 VRAM=38.7GiB | 12.6% done | ETA(epoch): 308s + [Epoch 37][60/66] loss=0.889009 avg=0.845910 VRAM=38.6GiB | 12.6% done | ETA(epoch): 115s + Train loss: 0.845978 (1269.2s) ETA: 6189min + Val loss: 0.863015 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9968 t_0.4-0.6=0.8449 t_0.6-0.8=0.7243 t_0.8-1.0=0.6954] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 38/299 (13% done) --- + [Epoch 38][10/66] loss=0.783078 avg=0.844263 VRAM=38.7GiB | 12.7% done | ETA(epoch): 1078s + [Epoch 38][20/66] loss=0.922807 avg=0.850094 VRAM=38.6GiB | 12.8% done | ETA(epoch): 885s + [Epoch 38][30/66] loss=0.865414 avg=0.860554 VRAM=38.7GiB | 12.8% done | ETA(epoch): 693s + [Epoch 38][40/66] loss=1.001537 avg=0.860855 VRAM=38.6GiB | 12.9% done | ETA(epoch): 501s + [Epoch 38][50/66] loss=0.887393 avg=0.861106 VRAM=38.7GiB | 12.9% done | ETA(epoch): 308s + [Epoch 38][60/66] loss=0.936230 avg=0.861682 VRAM=38.6GiB | 13.0% done | ETA(epoch): 116s + Train loss: 0.860625 (1270.7s) ETA: 6160min + Val loss: 0.847379 [t_0.0-0.2=1.0770 t_0.2-0.4=0.9787 t_0.4-0.6=0.8405 t_0.6-0.8=0.7447 t_0.8-1.0=0.6782] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 39/299 (13% done) --- + [Epoch 39][10/66] loss=0.870016 avg=0.862030 VRAM=38.7GiB | 13.1% done | ETA(epoch): 1079s + [Epoch 39][20/66] loss=0.782604 avg=0.856604 VRAM=38.6GiB | 13.1% done | ETA(epoch): 886s + [Epoch 39][30/66] loss=0.963964 avg=0.868365 VRAM=38.7GiB | 13.2% done | ETA(epoch): 693s + [Epoch 39][40/66] loss=0.778906 avg=0.860625 VRAM=38.6GiB | 13.2% done | ETA(epoch): 501s + [Epoch 39][50/66] loss=0.974998 avg=0.869290 VRAM=38.7GiB | 13.3% done | ETA(epoch): 308s + [Epoch 39][60/66] loss=0.737077 avg=0.865056 VRAM=38.6GiB | 13.3% done | ETA(epoch): 115s + Train loss: 0.863074 (1270.5s) ETA: 6131min + Val loss: 0.894089 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9632 t_0.4-0.6=0.8582 t_0.6-0.8=0.7419 t_0.8-1.0=0.6675] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 40/299 (13% done) --- + [Epoch 40][10/66] loss=0.945928 avg=0.906413 VRAM=38.7GiB | 13.4% done | ETA(epoch): 1077s + [Epoch 40][20/66] loss=0.821238 avg=0.902649 VRAM=38.6GiB | 13.4% done | ETA(epoch): 886s + [Epoch 40][30/66] loss=0.862399 avg=0.888947 VRAM=38.7GiB | 13.5% done | ETA(epoch): 694s + [Epoch 40][40/66] loss=0.869945 avg=0.875475 VRAM=38.6GiB | 13.5% done | ETA(epoch): 501s + [Epoch 40][50/66] loss=0.828056 avg=0.873298 VRAM=38.7GiB | 13.6% done | ETA(epoch): 308s + [Epoch 40][60/66] loss=0.909898 avg=0.880356 VRAM=38.6GiB | 13.6% done | ETA(epoch): 116s + Train loss: 0.878826 (1273.1s) ETA: 6102min + Val loss: 0.848590 [t_0.0-0.2=1.0864 t_0.2-0.4=0.9855 t_0.4-0.6=0.8372 t_0.6-0.8=0.7391 t_0.8-1.0=0.6786] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 41/299 (14% done) --- + [Epoch 41][10/66] loss=0.835537 avg=0.892545 VRAM=38.7GiB | 13.7% done | ETA(epoch): 1077s + [Epoch 41][20/66] loss=0.822478 avg=0.864383 VRAM=38.6GiB | 13.8% done | ETA(epoch): 885s + [Epoch 41][30/66] loss=0.742150 avg=0.857706 VRAM=38.7GiB | 13.8% done | ETA(epoch): 693s + [Epoch 41][40/66] loss=0.951247 avg=0.865851 VRAM=38.6GiB | 13.9% done | ETA(epoch): 501s + [Epoch 41][50/66] loss=0.957553 avg=0.863935 VRAM=38.7GiB | 13.9% done | ETA(epoch): 308s + [Epoch 41][60/66] loss=0.806437 avg=0.865193 VRAM=38.6GiB | 14.0% done | ETA(epoch): 115s + Train loss: 0.864840 (1270.0s) ETA: 6074min + Val loss: 0.848363 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9918 t_0.4-0.6=0.8424 t_0.6-0.8=0.7230 t_0.8-1.0=0.6668] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 18.9/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 42/299 (14% done) --- + [Epoch 42][10/66] loss=0.865123 avg=0.875499 VRAM=38.7GiB | 14.1% done | ETA(epoch): 1076s + [Epoch 42][20/66] loss=1.000747 avg=0.872725 VRAM=38.6GiB | 14.1% done | ETA(epoch): 884s + [Epoch 42][30/66] loss=0.840349 avg=0.866841 VRAM=38.7GiB | 14.2% done | ETA(epoch): 693s + [Epoch 42][40/66] loss=0.898146 avg=0.872407 VRAM=38.6GiB | 14.2% done | ETA(epoch): 502s + [Epoch 42][50/66] loss=0.727823 avg=0.875216 VRAM=38.7GiB | 14.3% done | ETA(epoch): 309s + [Epoch 42][60/66] loss=0.805595 avg=0.870678 VRAM=38.6GiB | 14.3% done | ETA(epoch): 116s + Train loss: 0.871699 (1272.5s) ETA: 6046min + Val loss: 0.831032 [t_0.0-0.2=1.0874 t_0.2-0.4=0.9978 t_0.4-0.6=0.8567 t_0.6-0.8=0.7239 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0042 (BEST) + Deleted old checkpoint: checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 43/299 (14% done) --- + [Epoch 43][10/66] loss=0.824401 avg=0.853581 VRAM=38.7GiB | 14.4% done | ETA(epoch): 1075s + [Epoch 43][20/66] loss=0.783473 avg=0.858295 VRAM=38.6GiB | 14.4% done | ETA(epoch): 884s + [Epoch 43][30/66] loss=0.971378 avg=0.866428 VRAM=38.7GiB | 14.5% done | ETA(epoch): 692s + [Epoch 43][40/66] loss=0.709895 avg=0.859954 VRAM=38.6GiB | 14.5% done | ETA(epoch): 500s + [Epoch 43][50/66] loss=0.945221 avg=0.865424 VRAM=38.7GiB | 14.6% done | ETA(epoch): 308s + [Epoch 43][60/66] loss=0.946425 avg=0.858666 VRAM=38.6GiB | 14.6% done | ETA(epoch): 115s + Train loss: 0.864482 (1269.8s) ETA: 6018min + Val loss: 0.885423 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9751 t_0.4-0.6=0.8904 t_0.6-0.8=0.6982 t_0.8-1.0=0.6710] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 44/299 (15% done) --- + [Epoch 44][10/66] loss=1.003111 avg=0.862830 VRAM=38.7GiB | 14.7% done | ETA(epoch): 1078s + [Epoch 44][20/66] loss=0.760244 avg=0.850591 VRAM=38.6GiB | 14.8% done | ETA(epoch): 886s + [Epoch 44][30/66] loss=0.931751 avg=0.851606 VRAM=38.7GiB | 14.8% done | ETA(epoch): 693s + [Epoch 44][40/66] loss=1.003587 avg=0.862035 VRAM=38.6GiB | 14.9% done | ETA(epoch): 501s + [Epoch 44][50/66] loss=0.791933 avg=0.868368 VRAM=38.7GiB | 14.9% done | ETA(epoch): 308s + [Epoch 44][60/66] loss=0.840676 avg=0.867028 VRAM=38.6GiB | 15.0% done | ETA(epoch): 116s + Train loss: 0.870048 (1270.9s) ETA: 5990min + Val loss: 0.869633 [t_0.0-0.2=1.0567 t_0.2-0.4=0.9706 t_0.4-0.6=0.8702 t_0.6-0.8=0.7061 t_0.8-1.0=0.6857] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 45/299 (15% done) --- + [Epoch 45][10/66] loss=0.876314 avg=0.823851 VRAM=38.7GiB | 15.1% done | ETA(epoch): 1087s + [Epoch 45][20/66] loss=0.839035 avg=0.827111 VRAM=38.6GiB | 15.1% done | ETA(epoch): 889s + [Epoch 45][30/66] loss=0.886505 avg=0.834530 VRAM=38.7GiB | 15.2% done | ETA(epoch): 695s + [Epoch 45][40/66] loss=1.025467 avg=0.854600 VRAM=38.6GiB | 15.2% done | ETA(epoch): 502s + [Epoch 45][50/66] loss=0.890408 avg=0.860433 VRAM=38.7GiB | 15.3% done | ETA(epoch): 308s + [Epoch 45][60/66] loss=0.908192 avg=0.857278 VRAM=38.6GiB | 15.3% done | ETA(epoch): 116s + Train loss: 0.857364 (1271.6s) ETA: 5963min + Val loss: 0.844574 [t_0.0-0.2=1.0927 t_0.2-0.4=1.0014 t_0.4-0.6=0.7710 t_0.6-0.8=0.7285 t_0.8-1.0=0.6560] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0045 +[MEM @ epoch 45 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 46/299 (15% done) --- + [Epoch 46][10/66] loss=0.899845 avg=0.897328 VRAM=38.7GiB | 15.4% done | ETA(epoch): 1077s + [Epoch 46][20/66] loss=0.920454 avg=0.899353 VRAM=38.6GiB | 15.4% done | ETA(epoch): 885s + [Epoch 46][30/66] loss=0.785645 avg=0.883766 VRAM=38.7GiB | 15.5% done | ETA(epoch): 693s + [Epoch 46][40/66] loss=0.951901 avg=0.881179 VRAM=38.6GiB | 15.5% done | ETA(epoch): 500s + [Epoch 46][50/66] loss=0.968659 avg=0.876344 VRAM=38.7GiB | 15.6% done | ETA(epoch): 308s + [Epoch 46][60/66] loss=0.890451 avg=0.868711 VRAM=38.6GiB | 15.6% done | ETA(epoch): 115s + Train loss: 0.867581 (1270.5s) ETA: 5936min + Val loss: 0.855299 [t_0.0-0.2=1.0634 t_0.2-0.4=0.9952 t_0.4-0.6=0.8604 t_0.6-0.8=0.7441 t_0.8-1.0=0.6719] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 47/299 (16% done) --- + [Epoch 47][10/66] loss=0.834137 avg=0.880383 VRAM=38.7GiB | 15.7% done | ETA(epoch): 1078s + [Epoch 47][20/66] loss=0.828840 avg=0.874851 VRAM=38.6GiB | 15.8% done | ETA(epoch): 885s + [Epoch 47][30/66] loss=0.971112 avg=0.885180 VRAM=38.7GiB | 15.8% done | ETA(epoch): 693s + [Epoch 47][40/66] loss=0.882694 avg=0.879683 VRAM=38.6GiB | 15.9% done | ETA(epoch): 500s + [Epoch 47][50/66] loss=0.770910 avg=0.877923 VRAM=38.7GiB | 15.9% done | ETA(epoch): 308s + [Epoch 47][60/66] loss=0.849860 avg=0.876243 VRAM=38.6GiB | 16.0% done | ETA(epoch): 116s + Train loss: 0.869960 (1270.9s) ETA: 5908min + Val loss: 0.839181 [t_0.0-0.2=1.0732 t_0.2-0.4=1.0430 t_0.4-0.6=0.8381 t_0.6-0.8=0.7034 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 48/299 (16% done) --- + [Epoch 48][10/66] loss=0.802799 avg=0.852966 VRAM=38.7GiB | 16.1% done | ETA(epoch): 1076s + [Epoch 48][20/66] loss=0.801301 avg=0.864358 VRAM=38.6GiB | 16.1% done | ETA(epoch): 885s + [Epoch 48][30/66] loss=0.849850 avg=0.865225 VRAM=38.7GiB | 16.2% done | ETA(epoch): 692s + [Epoch 48][40/66] loss=0.946426 avg=0.863103 VRAM=38.6GiB | 16.2% done | ETA(epoch): 500s + [Epoch 48][50/66] loss=0.765375 avg=0.857915 VRAM=38.7GiB | 16.3% done | ETA(epoch): 314s + [Epoch 48][60/66] loss=0.819871 avg=0.856119 VRAM=38.6GiB | 16.3% done | ETA(epoch): 117s + Train loss: 0.856915 (1287.8s) ETA: 5883min + Val loss: 0.872795 [t_0.0-0.2=1.0895 t_0.2-0.4=1.0219 t_0.4-0.6=0.8555 t_0.6-0.8=0.7355 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 49/299 (16% done) --- + [Epoch 49][10/66] loss=0.854359 avg=0.880775 VRAM=38.7GiB | 16.4% done | ETA(epoch): 1077s + [Epoch 49][20/66] loss=0.709925 avg=0.865383 VRAM=38.6GiB | 16.4% done | ETA(epoch): 885s + [Epoch 49][30/66] loss=0.964525 avg=0.866749 VRAM=38.7GiB | 16.5% done | ETA(epoch): 693s + [Epoch 49][40/66] loss=0.990060 avg=0.863193 VRAM=38.6GiB | 16.5% done | ETA(epoch): 500s + [Epoch 49][50/66] loss=0.836257 avg=0.857851 VRAM=38.7GiB | 16.6% done | ETA(epoch): 308s + [Epoch 49][60/66] loss=0.796590 avg=0.856469 VRAM=38.6GiB | 16.6% done | ETA(epoch): 115s + Train loss: 0.855136 (1269.7s) ETA: 5856min + Val loss: 0.866288 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9998 t_0.4-0.6=0.8505 t_0.6-0.8=0.7254 t_0.8-1.0=0.6752] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 50/299 (17% done) --- + [Epoch 50][10/66] loss=0.941054 avg=0.895049 VRAM=38.7GiB | 16.7% done | ETA(epoch): 1077s + [Epoch 50][20/66] loss=0.795341 avg=0.870521 VRAM=38.6GiB | 16.8% done | ETA(epoch): 885s + [Epoch 50][30/66] loss=0.779538 avg=0.868885 VRAM=38.7GiB | 16.8% done | ETA(epoch): 693s + [Epoch 50][40/66] loss=0.868154 avg=0.868095 VRAM=38.6GiB | 16.9% done | ETA(epoch): 501s + [Epoch 50][50/66] loss=0.853037 avg=0.873077 VRAM=38.7GiB | 16.9% done | ETA(epoch): 308s + [Epoch 50][60/66] loss=0.718780 avg=0.867109 VRAM=38.6GiB | 17.0% done | ETA(epoch): 115s + Train loss: 0.865646 (1270.3s) ETA: 5829min + Val loss: 0.828462 [t_0.0-0.2=1.0719 t_0.2-0.4=0.9887 t_0.4-0.6=0.8449 t_0.6-0.8=0.7336 t_0.8-1.0=0.6806] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0050 (BEST) + Deleted old checkpoint: checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 51/299 (17% done) --- + [Epoch 51][10/66] loss=0.780199 avg=0.832193 VRAM=38.7GiB | 17.1% done | ETA(epoch): 1077s + [Epoch 51][20/66] loss=0.908852 avg=0.856266 VRAM=38.6GiB | 17.1% done | ETA(epoch): 885s + [Epoch 51][30/66] loss=0.832762 avg=0.851540 VRAM=38.7GiB | 17.2% done | ETA(epoch): 692s + [Epoch 51][40/66] loss=0.858365 avg=0.865820 VRAM=38.6GiB | 17.2% done | ETA(epoch): 500s + [Epoch 51][50/66] loss=0.870828 avg=0.857469 VRAM=38.7GiB | 17.3% done | ETA(epoch): 308s + [Epoch 51][60/66] loss=0.832738 avg=0.859169 VRAM=38.6GiB | 17.3% done | ETA(epoch): 115s + Train loss: 0.861995 (1269.7s) ETA: 5803min + Val loss: 0.862244 [t_0.0-0.2=1.0577 t_0.2-0.4=1.0229 t_0.4-0.6=0.8069 t_0.6-0.8=0.7417 t_0.8-1.0=0.6764] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 52/299 (17% done) --- + [Epoch 52][10/66] loss=0.985605 avg=0.856762 VRAM=38.7GiB | 17.4% done | ETA(epoch): 1078s + [Epoch 52][20/66] loss=0.836705 avg=0.860141 VRAM=38.6GiB | 17.4% done | ETA(epoch): 891s + [Epoch 52][30/66] loss=0.795315 avg=0.864094 VRAM=38.7GiB | 17.5% done | ETA(epoch): 696s + [Epoch 52][40/66] loss=0.827862 avg=0.866239 VRAM=38.6GiB | 17.5% done | ETA(epoch): 502s + [Epoch 52][50/66] loss=0.832743 avg=0.864636 VRAM=38.7GiB | 17.6% done | ETA(epoch): 309s + [Epoch 52][60/66] loss=0.816108 avg=0.861671 VRAM=38.6GiB | 17.6% done | ETA(epoch): 116s + Train loss: 0.863665 (1273.6s) ETA: 5777min + Val loss: 0.872662 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9898 t_0.4-0.6=0.8639 t_0.6-0.8=0.7527 t_0.8-1.0=0.6736] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 53/299 (18% done) --- + [Epoch 53][10/66] loss=0.726901 avg=0.846719 VRAM=38.7GiB | 17.7% done | ETA(epoch): 1088s + [Epoch 53][20/66] loss=0.975186 avg=0.887363 VRAM=38.6GiB | 17.8% done | ETA(epoch): 889s + [Epoch 53][30/66] loss=0.924833 avg=0.894647 VRAM=38.7GiB | 17.8% done | ETA(epoch): 695s + [Epoch 53][40/66] loss=0.821693 avg=0.887079 VRAM=38.6GiB | 17.9% done | ETA(epoch): 502s + [Epoch 53][50/66] loss=0.930168 avg=0.883333 VRAM=38.7GiB | 17.9% done | ETA(epoch): 309s + [Epoch 53][60/66] loss=0.943530 avg=0.881196 VRAM=38.6GiB | 18.0% done | ETA(epoch): 116s + Train loss: 0.875666 (1272.6s) ETA: 5751min + Val loss: 0.880313 [t_0.0-0.2=1.0569 t_0.2-0.4=1.0359 t_0.4-0.6=0.8949 t_0.6-0.8=0.7162 t_0.8-1.0=0.6682] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0053 +[MEM @ epoch 53 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 54/299 (18% done) --- + [Epoch 54][10/66] loss=0.899863 avg=0.862677 VRAM=38.7GiB | 18.1% done | ETA(epoch): 1077s + [Epoch 54][20/66] loss=0.896571 avg=0.863927 VRAM=38.6GiB | 18.1% done | ETA(epoch): 886s + [Epoch 54][30/66] loss=1.023769 avg=0.854941 VRAM=38.7GiB | 18.2% done | ETA(epoch): 693s + [Epoch 54][40/66] loss=0.815146 avg=0.853353 VRAM=38.6GiB | 18.2% done | ETA(epoch): 501s + [Epoch 54][50/66] loss=0.842042 avg=0.852271 VRAM=38.7GiB | 18.3% done | ETA(epoch): 308s + [Epoch 54][60/66] loss=0.809314 avg=0.856695 VRAM=38.6GiB | 18.3% done | ETA(epoch): 116s + Train loss: 0.859389 (1270.6s) ETA: 5724min + Val loss: 0.871899 [t_0.0-0.2=1.0699 t_0.2-0.4=1.0087 t_0.4-0.6=0.8192 t_0.6-0.8=0.7262 t_0.8-1.0=0.6724] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 55/299 (18% done) --- + [Epoch 55][10/66] loss=0.900193 avg=0.850853 VRAM=38.7GiB | 18.4% done | ETA(epoch): 1077s + [Epoch 55][20/66] loss=0.818003 avg=0.876623 VRAM=38.6GiB | 18.4% done | ETA(epoch): 885s + [Epoch 55][30/66] loss=0.745577 avg=0.867755 VRAM=38.7GiB | 18.5% done | ETA(epoch): 692s + [Epoch 55][40/66] loss=0.829273 avg=0.870909 VRAM=38.6GiB | 18.5% done | ETA(epoch): 500s + [Epoch 55][50/66] loss=0.835825 avg=0.871862 VRAM=38.7GiB | 18.6% done | ETA(epoch): 308s + [Epoch 55][60/66] loss=0.744943 avg=0.866036 VRAM=38.6GiB | 18.6% done | ETA(epoch): 116s + Train loss: 0.862157 (1272.7s) ETA: 5699min + Val loss: 0.832167 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9681 t_0.4-0.6=0.8646 t_0.6-0.8=0.7281 t_0.8-1.0=0.6510] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 56/299 (19% done) --- + [Epoch 56][10/66] loss=0.804572 avg=0.843902 VRAM=38.7GiB | 18.7% done | ETA(epoch): 1077s + [Epoch 56][20/66] loss=1.023020 avg=0.852795 VRAM=38.6GiB | 18.8% done | ETA(epoch): 885s + [Epoch 56][30/66] loss=0.866054 avg=0.859278 VRAM=38.7GiB | 18.8% done | ETA(epoch): 693s + [Epoch 56][40/66] loss=0.767221 avg=0.855503 VRAM=38.6GiB | 18.9% done | ETA(epoch): 500s + [Epoch 56][50/66] loss=0.859547 avg=0.854365 VRAM=38.7GiB | 18.9% done | ETA(epoch): 308s + [Epoch 56][60/66] loss=0.848153 avg=0.857633 VRAM=38.6GiB | 19.0% done | ETA(epoch): 116s + Train loss: 0.858984 (1271.6s) ETA: 5673min + Val loss: 0.835374 [t_0.0-0.2=1.0825 t_0.2-0.4=1.0093 t_0.4-0.6=0.8448 t_0.6-0.8=0.7118 t_0.8-1.0=0.6660] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 57/299 (19% done) --- + [Epoch 57][10/66] loss=0.903999 avg=0.887422 VRAM=38.7GiB | 19.1% done | ETA(epoch): 1079s + [Epoch 57][20/66] loss=0.949385 avg=0.877620 VRAM=38.6GiB | 19.1% done | ETA(epoch): 886s + [Epoch 57][30/66] loss=0.912311 avg=0.871816 VRAM=38.7GiB | 19.2% done | ETA(epoch): 693s + [Epoch 57][40/66] loss=0.820575 avg=0.872672 VRAM=38.6GiB | 19.2% done | ETA(epoch): 500s + [Epoch 57][50/66] loss=0.808867 avg=0.872127 VRAM=38.7GiB | 19.3% done | ETA(epoch): 308s + [Epoch 57][60/66] loss=0.839187 avg=0.869507 VRAM=38.6GiB | 19.3% done | ETA(epoch): 115s + Train loss: 0.870155 (1269.5s) ETA: 5647min + Val loss: 0.874471 [t_0.0-0.2=1.0956 t_0.2-0.4=0.9886 t_0.4-0.6=0.8449 t_0.6-0.8=0.7041 t_0.8-1.0=0.6785] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 58/299 (19% done) --- + [Epoch 58][10/66] loss=0.918948 avg=0.861940 VRAM=38.7GiB | 19.4% done | ETA(epoch): 1075s + [Epoch 58][20/66] loss=0.794621 avg=0.873390 VRAM=38.6GiB | 19.4% done | ETA(epoch): 884s + [Epoch 58][30/66] loss=0.869971 avg=0.871447 VRAM=38.7GiB | 19.5% done | ETA(epoch): 692s + [Epoch 58][40/66] loss=0.953210 avg=0.874379 VRAM=38.6GiB | 19.5% done | ETA(epoch): 500s + [Epoch 58][50/66] loss=0.779447 avg=0.871432 VRAM=38.7GiB | 19.6% done | ETA(epoch): 307s + [Epoch 58][60/66] loss=0.835049 avg=0.882563 VRAM=38.6GiB | 19.6% done | ETA(epoch): 115s + Train loss: 0.881654 (1268.5s) ETA: 5621min + Val loss: 0.853108 [t_0.0-0.2=1.0551 t_0.2-0.4=1.0244 t_0.4-0.6=0.8371 t_0.6-0.8=0.7225 t_0.8-1.0=0.6580] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 59/299 (20% done) --- + [Epoch 59][10/66] loss=0.791811 avg=0.865715 VRAM=38.7GiB | 19.7% done | ETA(epoch): 1077s + [Epoch 59][20/66] loss=0.938470 avg=0.878653 VRAM=38.6GiB | 19.8% done | ETA(epoch): 889s + [Epoch 59][30/66] loss=0.784425 avg=0.871644 VRAM=38.7GiB | 19.8% done | ETA(epoch): 695s + [Epoch 59][40/66] loss=0.755391 avg=0.862041 VRAM=38.6GiB | 19.9% done | ETA(epoch): 501s + [Epoch 59][50/66] loss=0.922165 avg=0.862695 VRAM=38.7GiB | 19.9% done | ETA(epoch): 308s + [Epoch 59][60/66] loss=1.012689 avg=0.871540 VRAM=38.6GiB | 20.0% done | ETA(epoch): 116s + Train loss: 0.873254 (1271.5s) ETA: 5596min + Val loss: 0.885119 [t_0.0-0.2=1.0906 t_0.2-0.4=0.9846 t_0.4-0.6=0.8345 t_0.6-0.8=0.7555 t_0.8-1.0=0.7239] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 60/299 (20% done) --- + [MilestoneVis] train_0 step 3960 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3960 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3960 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3960 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3960 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3960 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3960 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3960 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3960 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3960 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 60][10/66] loss=0.738567 avg=0.866982 VRAM=38.7GiB | 20.1% done | ETA(epoch): 6788s + [Epoch 60][20/66] loss=0.945127 avg=0.855446 VRAM=38.6GiB | 20.1% done | ETA(epoch): 3230s + [Epoch 60][30/66] loss=0.974476 avg=0.857427 VRAM=38.7GiB | 20.2% done | ETA(epoch): 1916s + [Epoch 60][40/66] loss=0.879421 avg=0.864695 VRAM=38.6GiB | 20.2% done | ETA(epoch): 1163s + [Epoch 60][50/66] loss=0.881496 avg=0.864036 VRAM=38.7GiB | 20.3% done | ETA(epoch): 634s + [Epoch 60][60/66] loss=0.823659 avg=0.866769 VRAM=38.6GiB | 20.3% done | ETA(epoch): 217s + Train loss: 0.864427 (2289.3s) ETA: 5637min + Val loss: 0.876558 [t_0.0-0.2=1.0851 t_0.2-0.4=0.9743 t_0.4-0.6=0.8096 t_0.6-0.8=0.7417 t_0.8-1.0=0.6678] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 61/299 (20% done) --- + [Epoch 61][10/66] loss=0.797466 avg=0.870058 VRAM=38.7GiB | 20.4% done | ETA(epoch): 1077s + [Epoch 61][20/66] loss=0.896138 avg=0.874580 VRAM=38.6GiB | 20.4% done | ETA(epoch): 891s + [Epoch 61][30/66] loss=0.982956 avg=0.887469 VRAM=38.7GiB | 20.5% done | ETA(epoch): 696s + [Epoch 61][40/66] loss=0.870854 avg=0.878745 VRAM=38.6GiB | 20.5% done | ETA(epoch): 502s + [Epoch 61][50/66] loss=0.954906 avg=0.878294 VRAM=38.7GiB | 20.6% done | ETA(epoch): 309s + [Epoch 61][60/66] loss=0.966298 avg=0.878979 VRAM=38.6GiB | 20.6% done | ETA(epoch): 116s + Train loss: 0.878752 (1272.3s) ETA: 5610min + Val loss: 0.881016 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9966 t_0.4-0.6=0.8406 t_0.6-0.8=0.7454 t_0.8-1.0=0.6863] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 62/299 (21% done) --- + [Epoch 62][10/66] loss=0.834560 avg=0.870513 VRAM=38.7GiB | 20.7% done | ETA(epoch): 1076s + [Epoch 62][20/66] loss=0.837794 avg=0.852860 VRAM=38.6GiB | 20.8% done | ETA(epoch): 884s + [Epoch 62][30/66] loss=0.817476 avg=0.849449 VRAM=38.7GiB | 20.8% done | ETA(epoch): 692s + [Epoch 62][40/66] loss=0.928207 avg=0.860735 VRAM=38.6GiB | 20.9% done | ETA(epoch): 500s + [Epoch 62][50/66] loss=0.808046 avg=0.869399 VRAM=38.7GiB | 20.9% done | ETA(epoch): 308s + [Epoch 62][60/66] loss=0.734097 avg=0.865028 VRAM=38.6GiB | 21.0% done | ETA(epoch): 115s + Train loss: 0.869989 (1271.5s) ETA: 5583min + Val loss: 0.869568 [t_0.0-0.2=1.0790 t_0.2-0.4=1.0200 t_0.4-0.6=0.8681 t_0.6-0.8=0.7492 t_0.8-1.0=0.7092] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 63/299 (21% done) --- + [Epoch 63][10/66] loss=0.814997 avg=0.861362 VRAM=38.7GiB | 21.1% done | ETA(epoch): 1077s + [Epoch 63][20/66] loss=0.942825 avg=0.863492 VRAM=38.6GiB | 21.1% done | ETA(epoch): 884s + [Epoch 63][30/66] loss=0.968741 avg=0.875658 VRAM=38.7GiB | 21.2% done | ETA(epoch): 692s + [Epoch 63][40/66] loss=0.863312 avg=0.883899 VRAM=38.6GiB | 21.2% done | ETA(epoch): 500s + [Epoch 63][50/66] loss=0.931591 avg=0.885084 VRAM=38.7GiB | 21.3% done | ETA(epoch): 308s + [Epoch 63][60/66] loss=0.816447 avg=0.885744 VRAM=38.6GiB | 21.3% done | ETA(epoch): 115s + Train loss: 0.885398 (1268.9s) ETA: 5557min + Val loss: 0.901580 [t_0.0-0.2=1.0870 t_0.2-0.4=1.0097 t_0.4-0.6=0.8401 t_0.6-0.8=0.7297 t_0.8-1.0=0.6927] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 64/299 (21% done) --- + [Epoch 64][10/66] loss=0.871387 avg=0.876068 VRAM=38.7GiB | 21.4% done | ETA(epoch): 1075s + [Epoch 64][20/66] loss=0.934077 avg=0.884484 VRAM=38.6GiB | 21.4% done | ETA(epoch): 884s + [Epoch 64][30/66] loss=0.977062 avg=0.889343 VRAM=38.7GiB | 21.5% done | ETA(epoch): 692s + [Epoch 64][40/66] loss=0.922428 avg=0.875288 VRAM=38.6GiB | 21.5% done | ETA(epoch): 500s + [Epoch 64][50/66] loss=0.828015 avg=0.870147 VRAM=38.7GiB | 21.6% done | ETA(epoch): 309s + [Epoch 64][60/66] loss=0.828576 avg=0.866758 VRAM=38.6GiB | 21.6% done | ETA(epoch): 116s + Train loss: 0.867653 (1271.6s) ETA: 5530min + Val loss: 0.904507 [t_0.0-0.2=1.0827 t_0.2-0.4=0.9947 t_0.4-0.6=0.8673 t_0.6-0.8=0.7354 t_0.8-1.0=0.6722] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 65/299 (22% done) --- + [Epoch 65][10/66] loss=0.827862 avg=0.847838 VRAM=38.7GiB | 21.7% done | ETA(epoch): 1076s + [Epoch 65][20/66] loss=0.787536 avg=0.853681 VRAM=38.6GiB | 21.8% done | ETA(epoch): 884s + [Epoch 65][30/66] loss=0.756832 avg=0.854002 VRAM=38.7GiB | 21.8% done | ETA(epoch): 692s + [Epoch 65][40/66] loss=0.751822 avg=0.854712 VRAM=38.6GiB | 21.9% done | ETA(epoch): 500s + [Epoch 65][50/66] loss=0.947470 avg=0.855022 VRAM=38.7GiB | 21.9% done | ETA(epoch): 308s + [Epoch 65][60/66] loss=0.933171 avg=0.862199 VRAM=38.6GiB | 22.0% done | ETA(epoch): 115s + Train loss: 0.865428 (1269.0s) ETA: 5504min + Val loss: 0.886220 [t_0.0-0.2=1.0824 t_0.2-0.4=0.9875 t_0.4-0.6=0.8400 t_0.6-0.8=0.7145 t_0.8-1.0=0.6730] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 66/299 (22% done) --- + [Epoch 66][10/66] loss=0.879295 avg=0.863341 VRAM=38.7GiB | 22.1% done | ETA(epoch): 1076s + [Epoch 66][20/66] loss=0.820807 avg=0.847751 VRAM=38.6GiB | 22.1% done | ETA(epoch): 885s + [Epoch 66][30/66] loss=0.788207 avg=0.853645 VRAM=38.7GiB | 22.2% done | ETA(epoch): 693s + [Epoch 66][40/66] loss=0.771600 avg=0.853928 VRAM=38.6GiB | 22.2% done | ETA(epoch): 500s + [Epoch 66][50/66] loss=0.947684 avg=0.855410 VRAM=38.7GiB | 22.3% done | ETA(epoch): 308s + [Epoch 66][60/66] loss=0.873169 avg=0.860014 VRAM=38.6GiB | 22.3% done | ETA(epoch): 115s + Train loss: 0.859977 (1269.6s) ETA: 5478min + Val loss: 0.872971 [t_0.0-0.2=1.0781 t_0.2-0.4=1.0008 t_0.4-0.6=0.8305 t_0.6-0.8=0.7146 t_0.8-1.0=0.6597] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 67/299 (22% done) --- + [Epoch 67][10/66] loss=0.812912 avg=0.817395 VRAM=38.7GiB | 22.4% done | ETA(epoch): 1076s + [Epoch 67][20/66] loss=0.946600 avg=0.852307 VRAM=38.6GiB | 22.4% done | ETA(epoch): 885s + [Epoch 67][30/66] loss=0.843191 avg=0.868835 VRAM=38.7GiB | 22.5% done | ETA(epoch): 696s + [Epoch 67][40/66] loss=0.828769 avg=0.863215 VRAM=38.6GiB | 22.5% done | ETA(epoch): 502s + [Epoch 67][50/66] loss=0.912813 avg=0.857988 VRAM=38.7GiB | 22.6% done | ETA(epoch): 309s + [Epoch 67][60/66] loss=0.962461 avg=0.866152 VRAM=38.6GiB | 22.6% done | ETA(epoch): 116s + Train loss: 0.865001 (1272.3s) ETA: 5452min + Val loss: 0.863588 [t_0.0-0.2=1.0840 t_0.2-0.4=1.0174 t_0.4-0.6=0.8839 t_0.6-0.8=0.6909 t_0.8-1.0=0.6712] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 68/299 (23% done) --- + [Epoch 68][10/66] loss=0.879576 avg=0.887520 VRAM=38.7GiB | 22.7% done | ETA(epoch): 1079s + [Epoch 68][20/66] loss=0.838413 avg=0.862305 VRAM=38.6GiB | 22.8% done | ETA(epoch): 894s + [Epoch 68][30/66] loss=0.751089 avg=0.864388 VRAM=38.7GiB | 22.8% done | ETA(epoch): 697s + [Epoch 68][40/66] loss=0.802494 avg=0.864899 VRAM=38.6GiB | 22.9% done | ETA(epoch): 503s + [Epoch 68][50/66] loss=0.961741 avg=0.862424 VRAM=38.7GiB | 22.9% done | ETA(epoch): 309s + [Epoch 68][60/66] loss=0.911101 avg=0.860241 VRAM=38.6GiB | 23.0% done | ETA(epoch): 116s + Train loss: 0.855414 (1273.8s) ETA: 5426min + Val loss: 0.902222 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9847 t_0.4-0.6=0.8594 t_0.6-0.8=0.7378 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 69/299 (23% done) --- + [Epoch 69][10/66] loss=0.825566 avg=0.847992 VRAM=38.7GiB | 23.1% done | ETA(epoch): 1088s + [Epoch 69][20/66] loss=0.866260 avg=0.832097 VRAM=38.6GiB | 23.1% done | ETA(epoch): 889s + [Epoch 69][30/66] loss=0.894633 avg=0.853216 VRAM=38.7GiB | 23.2% done | ETA(epoch): 695s + [Epoch 69][40/66] loss=0.763925 avg=0.846395 VRAM=38.6GiB | 23.2% done | ETA(epoch): 502s + [Epoch 69][50/66] loss=0.859609 avg=0.842492 VRAM=38.7GiB | 23.3% done | ETA(epoch): 308s + [Epoch 69][60/66] loss=0.870558 avg=0.843434 VRAM=38.6GiB | 23.3% done | ETA(epoch): 116s + Train loss: 0.839921 (1271.5s) ETA: 5400min + Val loss: 0.858354 [t_0.0-0.2=1.0717 t_0.2-0.4=1.0265 t_0.4-0.6=0.8616 t_0.6-0.8=0.7111 t_0.8-1.0=0.6644] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 70/299 (23% done) --- + [Epoch 70][10/66] loss=0.988185 avg=0.871180 VRAM=38.7GiB | 23.4% done | ETA(epoch): 1075s + [Epoch 70][20/66] loss=0.992854 avg=0.861088 VRAM=38.6GiB | 23.4% done | ETA(epoch): 884s + [Epoch 70][30/66] loss=0.976837 avg=0.870503 VRAM=38.7GiB | 23.5% done | ETA(epoch): 691s + [Epoch 70][40/66] loss=0.872543 avg=0.869211 VRAM=38.6GiB | 23.5% done | ETA(epoch): 500s + [Epoch 70][50/66] loss=0.878766 avg=0.864700 VRAM=38.7GiB | 23.6% done | ETA(epoch): 307s + [Epoch 70][60/66] loss=0.953961 avg=0.861539 VRAM=38.6GiB | 23.6% done | ETA(epoch): 115s + Train loss: 0.862872 (1270.4s) ETA: 5374min + Val loss: 0.869600 [t_0.0-0.2=1.0591 t_0.2-0.4=0.9774 t_0.4-0.6=0.8628 t_0.6-0.8=0.7043 t_0.8-1.0=0.6754] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 71/299 (24% done) --- + [Epoch 71][10/66] loss=0.935934 avg=0.869364 VRAM=38.7GiB | 23.7% done | ETA(epoch): 1075s + [Epoch 71][20/66] loss=0.862215 avg=0.867337 VRAM=38.6GiB | 23.8% done | ETA(epoch): 883s + [Epoch 71][30/66] loss=0.838125 avg=0.868049 VRAM=38.7GiB | 23.8% done | ETA(epoch): 691s + [Epoch 71][40/66] loss=0.940471 avg=0.859701 VRAM=38.6GiB | 23.9% done | ETA(epoch): 500s + [Epoch 71][50/66] loss=0.994460 avg=0.862927 VRAM=38.7GiB | 23.9% done | ETA(epoch): 308s + [Epoch 71][60/66] loss=0.866770 avg=0.862585 VRAM=38.6GiB | 24.0% done | ETA(epoch): 116s + Train loss: 0.862759 (1270.5s) ETA: 5349min + Val loss: 0.874374 [t_0.0-0.2=1.0805 t_0.2-0.4=1.0064 t_0.4-0.6=0.8225 t_0.6-0.8=0.7314 t_0.8-1.0=0.6737] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 72/299 (24% done) --- + [Epoch 72][10/66] loss=0.861008 avg=0.880197 VRAM=38.7GiB | 24.1% done | ETA(epoch): 1076s + [Epoch 72][20/66] loss=0.975805 avg=0.862456 VRAM=38.6GiB | 24.1% done | ETA(epoch): 884s + [Epoch 72][30/66] loss=0.820870 avg=0.860265 VRAM=38.7GiB | 24.2% done | ETA(epoch): 692s + [Epoch 72][40/66] loss=0.925401 avg=0.863037 VRAM=38.6GiB | 24.2% done | ETA(epoch): 500s + [Epoch 72][50/66] loss=0.856332 avg=0.865938 VRAM=38.7GiB | 24.3% done | ETA(epoch): 309s + [Epoch 72][60/66] loss=0.934936 avg=0.869965 VRAM=38.6GiB | 24.3% done | ETA(epoch): 116s + Train loss: 0.869203 (1272.9s) ETA: 5323min + Val loss: 0.847792 [t_0.0-0.2=1.0786 t_0.2-0.4=0.9847 t_0.4-0.6=0.8688 t_0.6-0.8=0.7395 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 73/299 (24% done) --- + [Epoch 73][10/66] loss=0.741829 avg=0.834901 VRAM=38.7GiB | 24.4% done | ETA(epoch): 1074s + [Epoch 73][20/66] loss=0.895536 avg=0.834492 VRAM=38.6GiB | 24.4% done | ETA(epoch): 883s + [Epoch 73][30/66] loss=0.857618 avg=0.846059 VRAM=38.7GiB | 24.5% done | ETA(epoch): 691s + [Epoch 73][40/66] loss=0.936665 avg=0.848717 VRAM=38.6GiB | 24.5% done | ETA(epoch): 502s + [Epoch 73][50/66] loss=0.913613 avg=0.846574 VRAM=38.7GiB | 24.6% done | ETA(epoch): 309s + [Epoch 73][60/66] loss=0.899269 avg=0.845670 VRAM=38.6GiB | 24.6% done | ETA(epoch): 116s + Train loss: 0.850074 (1272.8s) ETA: 5298min + Val loss: 0.859507 [t_0.0-0.2=1.0756 t_0.2-0.4=1.0133 t_0.4-0.6=0.8770 t_0.6-0.8=0.7128 t_0.8-1.0=0.6527] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 74/299 (25% done) --- + [Epoch 74][10/66] loss=0.718505 avg=0.843603 VRAM=38.7GiB | 24.7% done | ETA(epoch): 1077s + [Epoch 74][20/66] loss=0.750713 avg=0.847847 VRAM=38.6GiB | 24.8% done | ETA(epoch): 884s + [Epoch 74][30/66] loss=0.962389 avg=0.859890 VRAM=38.7GiB | 24.8% done | ETA(epoch): 713s + [Epoch 74][40/66] loss=0.823883 avg=0.855494 VRAM=38.6GiB | 24.9% done | ETA(epoch): 511s + [Epoch 74][50/66] loss=0.867808 avg=0.854826 VRAM=38.7GiB | 24.9% done | ETA(epoch): 313s + [Epoch 74][60/66] loss=0.791953 avg=0.858014 VRAM=38.6GiB | 25.0% done | ETA(epoch): 117s + Train loss: 0.853648 (1287.1s) ETA: 5273min + Val loss: 0.899947 [t_0.0-0.2=1.0774 t_0.2-0.4=1.0058 t_0.4-0.6=0.8571 t_0.6-0.8=0.7361 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 75/299 (25% done) --- + [Epoch 75][10/66] loss=0.859433 avg=0.842057 VRAM=38.7GiB | 25.1% done | ETA(epoch): 1077s + [Epoch 75][20/66] loss=0.843725 avg=0.861940 VRAM=38.6GiB | 25.1% done | ETA(epoch): 885s + [Epoch 75][30/66] loss=0.762932 avg=0.860525 VRAM=38.7GiB | 25.2% done | ETA(epoch): 692s + [Epoch 75][40/66] loss=0.851173 avg=0.855586 VRAM=38.6GiB | 25.2% done | ETA(epoch): 500s + [Epoch 75][50/66] loss=0.997419 avg=0.865161 VRAM=38.7GiB | 25.3% done | ETA(epoch): 307s + [Epoch 75][60/66] loss=0.947841 avg=0.861194 VRAM=38.6GiB | 25.3% done | ETA(epoch): 115s + Train loss: 0.859227 (1268.5s) ETA: 5248min + Val loss: 0.834811 [t_0.0-0.2=1.0469 t_0.2-0.4=1.0432 t_0.4-0.6=0.8614 t_0.6-0.8=0.7245 t_0.8-1.0=0.6584] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 76/299 (25% done) --- + [Epoch 76][10/66] loss=0.885076 avg=0.888568 VRAM=38.7GiB | 25.4% done | ETA(epoch): 1075s + [Epoch 76][20/66] loss=0.794617 avg=0.876345 VRAM=38.6GiB | 25.4% done | ETA(epoch): 898s + [Epoch 76][30/66] loss=0.779956 avg=0.871980 VRAM=38.7GiB | 25.5% done | ETA(epoch): 699s + [Epoch 76][40/66] loss=0.941668 avg=0.862116 VRAM=38.6GiB | 25.5% done | ETA(epoch): 504s + [Epoch 76][50/66] loss=0.793881 avg=0.862314 VRAM=38.7GiB | 25.6% done | ETA(epoch): 309s + [Epoch 76][60/66] loss=0.756490 avg=0.863846 VRAM=38.6GiB | 25.6% done | ETA(epoch): 116s + Train loss: 0.865863 (1274.6s) ETA: 5222min + Val loss: 0.889505 [t_0.0-0.2=1.0900 t_0.2-0.4=0.9935 t_0.4-0.6=0.8542 t_0.6-0.8=0.7389 t_0.8-1.0=0.6550] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 77/299 (26% done) --- + [Epoch 77][10/66] loss=0.814564 avg=0.877046 VRAM=38.7GiB | 25.7% done | ETA(epoch): 1076s + [Epoch 77][20/66] loss=0.866348 avg=0.854217 VRAM=38.6GiB | 25.8% done | ETA(epoch): 883s + [Epoch 77][30/66] loss=0.912243 avg=0.864038 VRAM=38.7GiB | 25.8% done | ETA(epoch): 691s + [Epoch 77][40/66] loss=0.930006 avg=0.865184 VRAM=38.6GiB | 25.9% done | ETA(epoch): 499s + [Epoch 77][50/66] loss=0.831677 avg=0.859814 VRAM=38.7GiB | 25.9% done | ETA(epoch): 307s + [Epoch 77][60/66] loss=0.846731 avg=0.864198 VRAM=38.6GiB | 26.0% done | ETA(epoch): 115s + Train loss: 0.866652 (1269.5s) ETA: 5197min + Val loss: 0.884841 [t_0.0-0.2=1.0839 t_0.2-0.4=0.9942 t_0.4-0.6=0.8644 t_0.6-0.8=0.7069 t_0.8-1.0=0.6664] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 78/299 (26% done) --- + [Epoch 78][10/66] loss=0.710875 avg=0.846148 VRAM=38.7GiB | 26.1% done | ETA(epoch): 1076s + [Epoch 78][20/66] loss=0.831635 avg=0.835495 VRAM=38.6GiB | 26.1% done | ETA(epoch): 885s + [Epoch 78][30/66] loss=0.884655 avg=0.844981 VRAM=38.7GiB | 26.2% done | ETA(epoch): 693s + [Epoch 78][40/66] loss=0.793023 avg=0.849417 VRAM=38.6GiB | 26.2% done | ETA(epoch): 500s + [Epoch 78][50/66] loss=0.857578 avg=0.853295 VRAM=38.7GiB | 26.3% done | ETA(epoch): 308s + [Epoch 78][60/66] loss=0.796980 avg=0.859461 VRAM=38.6GiB | 26.3% done | ETA(epoch): 116s + Train loss: 0.860248 (1276.1s) ETA: 5172min + Val loss: 0.857455 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0064 t_0.4-0.6=0.8433 t_0.6-0.8=0.7338 t_0.8-1.0=0.6744] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 79/299 (26% done) --- + [Epoch 79][10/66] loss=0.786471 avg=0.897047 VRAM=38.7GiB | 26.4% done | ETA(epoch): 1076s + [Epoch 79][20/66] loss=0.842073 avg=0.883677 VRAM=38.6GiB | 26.4% done | ETA(epoch): 884s + [Epoch 79][30/66] loss=0.846776 avg=0.864862 VRAM=38.7GiB | 26.5% done | ETA(epoch): 692s + [Epoch 79][40/66] loss=0.796454 avg=0.853616 VRAM=38.6GiB | 26.5% done | ETA(epoch): 500s + [Epoch 79][50/66] loss=0.777696 avg=0.847084 VRAM=38.7GiB | 26.6% done | ETA(epoch): 308s + [Epoch 79][60/66] loss=0.915121 avg=0.850100 VRAM=38.6GiB | 26.6% done | ETA(epoch): 115s + Train loss: 0.853622 (1270.2s) ETA: 5147min + Val loss: 0.872057 [t_0.0-0.2=1.0580 t_0.2-0.4=0.9958 t_0.4-0.6=0.8595 t_0.6-0.8=0.7275 t_0.8-1.0=0.6574] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 80/299 (27% done) --- + [Epoch 80][10/66] loss=0.894971 avg=0.907434 VRAM=38.7GiB | 26.7% done | ETA(epoch): 1074s + [Epoch 80][20/66] loss=0.903653 avg=0.881259 VRAM=38.6GiB | 26.8% done | ETA(epoch): 882s + [Epoch 80][30/66] loss=0.921066 avg=0.867031 VRAM=38.7GiB | 26.8% done | ETA(epoch): 691s + [Epoch 80][40/66] loss=0.841665 avg=0.847211 VRAM=38.6GiB | 26.9% done | ETA(epoch): 499s + [Epoch 80][50/66] loss=0.778411 avg=0.860267 VRAM=38.7GiB | 26.9% done | ETA(epoch): 307s + [Epoch 80][60/66] loss=0.773638 avg=0.856862 VRAM=38.6GiB | 27.0% done | ETA(epoch): 115s + Train loss: 0.858388 (1267.8s) ETA: 5122min + Val loss: 0.838243 [t_0.0-0.2=1.0694 t_0.2-0.4=1.0163 t_0.4-0.6=0.8536 t_0.6-0.8=0.7291 t_0.8-1.0=0.6598] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 81/299 (27% done) --- + [Epoch 81][10/66] loss=0.873234 avg=0.862010 VRAM=38.7GiB | 27.1% done | ETA(epoch): 1075s + [Epoch 81][20/66] loss=0.829197 avg=0.857560 VRAM=38.6GiB | 27.1% done | ETA(epoch): 883s + [Epoch 81][30/66] loss=0.749396 avg=0.840661 VRAM=38.7GiB | 27.2% done | ETA(epoch): 691s + [Epoch 81][40/66] loss=0.943186 avg=0.858032 VRAM=38.6GiB | 27.2% done | ETA(epoch): 499s + [Epoch 81][50/66] loss=0.897852 avg=0.864724 VRAM=38.7GiB | 27.3% done | ETA(epoch): 307s + [Epoch 81][60/66] loss=0.823745 avg=0.863882 VRAM=38.6GiB | 27.3% done | ETA(epoch): 115s + Train loss: 0.861922 (1267.4s) ETA: 5096min + Val loss: 0.887448 [t_0.0-0.2=1.0809 t_0.2-0.4=0.9819 t_0.4-0.6=0.8540 t_0.6-0.8=0.7278 t_0.8-1.0=0.6495] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 82/299 (27% done) --- + [Epoch 82][10/66] loss=0.903860 avg=0.862976 VRAM=38.7GiB | 27.4% done | ETA(epoch): 1074s + [Epoch 82][20/66] loss=0.868579 avg=0.861188 VRAM=38.6GiB | 27.4% done | ETA(epoch): 883s + [Epoch 82][30/66] loss=0.774085 avg=0.848553 VRAM=38.7GiB | 27.5% done | ETA(epoch): 691s + [Epoch 82][40/66] loss=0.885742 avg=0.852228 VRAM=38.6GiB | 27.5% done | ETA(epoch): 499s + [Epoch 82][50/66] loss=0.911791 avg=0.856258 VRAM=38.7GiB | 27.6% done | ETA(epoch): 307s + [Epoch 82][60/66] loss=0.878034 avg=0.851299 VRAM=38.6GiB | 27.6% done | ETA(epoch): 115s + Train loss: 0.848567 (1267.7s) ETA: 5071min + Val loss: 0.873979 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0074 t_0.4-0.6=0.8484 t_0.6-0.8=0.7226 t_0.8-1.0=0.6773] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 83/299 (28% done) --- + [Epoch 83][10/66] loss=0.768116 avg=0.844077 VRAM=38.7GiB | 27.7% done | ETA(epoch): 1074s + [Epoch 83][20/66] loss=0.803401 avg=0.857848 VRAM=38.6GiB | 27.8% done | ETA(epoch): 887s + [Epoch 83][30/66] loss=0.945723 avg=0.871688 VRAM=38.7GiB | 27.8% done | ETA(epoch): 693s + [Epoch 83][40/66] loss=0.862625 avg=0.866637 VRAM=38.6GiB | 27.9% done | ETA(epoch): 500s + [Epoch 83][50/66] loss=0.782844 avg=0.867841 VRAM=38.7GiB | 27.9% done | ETA(epoch): 308s + [Epoch 83][60/66] loss=0.705502 avg=0.867656 VRAM=38.6GiB | 28.0% done | ETA(epoch): 115s + Train loss: 0.870977 (1269.2s) ETA: 5046min + Val loss: 0.841910 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0064 t_0.4-0.6=0.8446 t_0.6-0.8=0.7314 t_0.8-1.0=0.6721] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 84/299 (28% done) --- + [Epoch 84][10/66] loss=0.727832 avg=0.811222 VRAM=38.7GiB | 28.1% done | ETA(epoch): 1084s + [Epoch 84][20/66] loss=0.896837 avg=0.831837 VRAM=38.6GiB | 28.1% done | ETA(epoch): 887s + [Epoch 84][30/66] loss=0.859323 avg=0.838342 VRAM=38.7GiB | 28.2% done | ETA(epoch): 693s + [Epoch 84][40/66] loss=0.839562 avg=0.843073 VRAM=38.6GiB | 28.2% done | ETA(epoch): 501s + [Epoch 84][50/66] loss=0.968622 avg=0.846769 VRAM=38.7GiB | 28.3% done | ETA(epoch): 308s + [Epoch 84][60/66] loss=0.982684 avg=0.852260 VRAM=38.6GiB | 28.3% done | ETA(epoch): 115s + Train loss: 0.851944 (1269.4s) ETA: 5021min + Val loss: 0.838245 [t_0.0-0.2=1.0858 t_0.2-0.4=1.0058 t_0.4-0.6=0.8366 t_0.6-0.8=0.7011 t_0.8-1.0=0.6574] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 85/299 (28% done) --- + [Epoch 85][10/66] loss=0.894185 avg=0.896208 VRAM=38.7GiB | 28.4% done | ETA(epoch): 1074s + [Epoch 85][20/66] loss=0.807206 avg=0.875948 VRAM=38.6GiB | 28.4% done | ETA(epoch): 884s + [Epoch 85][30/66] loss=0.835396 avg=0.868214 VRAM=38.7GiB | 28.5% done | ETA(epoch): 691s + [Epoch 85][40/66] loss=0.760288 avg=0.865697 VRAM=38.6GiB | 28.5% done | ETA(epoch): 499s + [Epoch 85][50/66] loss=0.871349 avg=0.863910 VRAM=38.7GiB | 28.6% done | ETA(epoch): 307s + [Epoch 85][60/66] loss=0.826326 avg=0.859822 VRAM=38.6GiB | 28.6% done | ETA(epoch): 115s + Train loss: 0.858241 (1270.7s) ETA: 4996min + Val loss: 0.856389 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0004 t_0.4-0.6=0.8279 t_0.6-0.8=0.7024 t_0.8-1.0=0.6639] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 86/299 (29% done) --- + [Epoch 86][10/66] loss=0.754760 avg=0.858773 VRAM=38.7GiB | 28.7% done | ETA(epoch): 1074s + [Epoch 86][20/66] loss=0.940001 avg=0.860020 VRAM=38.6GiB | 28.8% done | ETA(epoch): 883s + [Epoch 86][30/66] loss=0.957542 avg=0.870207 VRAM=38.7GiB | 28.8% done | ETA(epoch): 691s + [Epoch 86][40/66] loss=0.901875 avg=0.865632 VRAM=38.6GiB | 28.9% done | ETA(epoch): 499s + [Epoch 86][50/66] loss=0.880954 avg=0.867079 VRAM=38.7GiB | 28.9% done | ETA(epoch): 307s + [Epoch 86][60/66] loss=0.772496 avg=0.864340 VRAM=38.6GiB | 29.0% done | ETA(epoch): 115s + Train loss: 0.860583 (1267.1s) ETA: 4971min + Val loss: 0.873477 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9778 t_0.4-0.6=0.8203 t_0.6-0.8=0.7249 t_0.8-1.0=0.6568] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 87/299 (29% done) --- + [Epoch 87][10/66] loss=0.897769 avg=0.883379 VRAM=38.7GiB | 29.1% done | ETA(epoch): 1073s + [Epoch 87][20/66] loss=0.751162 avg=0.876371 VRAM=38.6GiB | 29.1% done | ETA(epoch): 882s + [Epoch 87][30/66] loss=0.933010 avg=0.880757 VRAM=38.7GiB | 29.2% done | ETA(epoch): 690s + [Epoch 87][40/66] loss=0.799140 avg=0.876650 VRAM=38.6GiB | 29.2% done | ETA(epoch): 499s + [Epoch 87][50/66] loss=0.771625 avg=0.872668 VRAM=38.7GiB | 29.3% done | ETA(epoch): 307s + [Epoch 87][60/66] loss=0.773958 avg=0.868262 VRAM=38.6GiB | 29.3% done | ETA(epoch): 115s + Train loss: 0.864805 (1267.1s) ETA: 4947min + Val loss: 0.843520 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0035 t_0.4-0.6=0.8366 t_0.6-0.8=0.7354 t_0.8-1.0=0.6623] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 88/299 (29% done) --- + [Epoch 88][10/66] loss=0.768809 avg=0.897260 VRAM=38.7GiB | 29.4% done | ETA(epoch): 1075s + [Epoch 88][20/66] loss=0.877831 avg=0.897041 VRAM=38.6GiB | 29.4% done | ETA(epoch): 883s + [Epoch 88][30/66] loss=0.899738 avg=0.894098 VRAM=38.7GiB | 29.5% done | ETA(epoch): 691s + [Epoch 88][40/66] loss=0.714545 avg=0.890255 VRAM=38.6GiB | 29.5% done | ETA(epoch): 499s + [Epoch 88][50/66] loss=0.849110 avg=0.889926 VRAM=38.7GiB | 29.6% done | ETA(epoch): 307s + [Epoch 88][60/66] loss=0.940232 avg=0.883673 VRAM=38.6GiB | 29.6% done | ETA(epoch): 115s + Train loss: 0.880405 (1267.1s) ETA: 4922min + Val loss: 0.833310 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9613 t_0.4-0.6=0.8686 t_0.6-0.8=0.7304 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 89/299 (30% done) --- + [Epoch 89][10/66] loss=0.822786 avg=0.894006 VRAM=38.7GiB | 29.7% done | ETA(epoch): 1075s + [Epoch 89][20/66] loss=0.962482 avg=0.896169 VRAM=38.6GiB | 29.8% done | ETA(epoch): 883s + [Epoch 89][30/66] loss=0.911320 avg=0.894924 VRAM=38.7GiB | 29.8% done | ETA(epoch): 691s + [Epoch 89][40/66] loss=0.815756 avg=0.885896 VRAM=38.6GiB | 29.9% done | ETA(epoch): 499s + [Epoch 89][50/66] loss=0.891877 avg=0.880757 VRAM=38.7GiB | 29.9% done | ETA(epoch): 307s + [Epoch 89][60/66] loss=0.858401 avg=0.876955 VRAM=38.6GiB | 30.0% done | ETA(epoch): 115s + Train loss: 0.871604 (1268.1s) ETA: 4897min + Val loss: 0.874427 [t_0.0-0.2=1.0889 t_0.2-0.4=0.9860 t_0.4-0.6=0.8590 t_0.6-0.8=0.7259 t_0.8-1.0=0.6631] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 90/299 (30% done) --- + [MilestoneVis] train_0 step 5940 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 5940 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 5940 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 5940 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 5940 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 5940 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 5940 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 5940 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 5940 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 5940 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 90][10/66] loss=0.759873 avg=0.834812 VRAM=38.7GiB | 30.1% done | ETA(epoch): 6943s + [Epoch 90][20/66] loss=1.018129 avg=0.860174 VRAM=38.6GiB | 30.1% done | ETA(epoch): 3293s + [Epoch 90][30/66] loss=0.859460 avg=0.859537 VRAM=38.7GiB | 30.2% done | ETA(epoch): 1949s + [Epoch 90][40/66] loss=0.826952 avg=0.859808 VRAM=38.6GiB | 30.2% done | ETA(epoch): 1181s + [Epoch 90][50/66] loss=0.894607 avg=0.857245 VRAM=38.7GiB | 30.3% done | ETA(epoch): 643s + [Epoch 90][60/66] loss=0.933093 avg=0.860184 VRAM=38.6GiB | 30.3% done | ETA(epoch): 220s + Train loss: 0.856035 (2315.6s) ETA: 4912min + Val loss: 0.848417 [t_0.0-0.2=1.0794 t_0.2-0.4=1.0067 t_0.4-0.6=0.8774 t_0.6-0.8=0.7273 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 91/299 (30% done) --- + [Epoch 91][10/66] loss=0.737765 avg=0.853549 VRAM=38.7GiB | 30.4% done | ETA(epoch): 1073s + [Epoch 91][20/66] loss=0.827524 avg=0.857564 VRAM=38.6GiB | 30.4% done | ETA(epoch): 884s + [Epoch 91][30/66] loss=0.895859 avg=0.849995 VRAM=38.7GiB | 30.5% done | ETA(epoch): 692s + [Epoch 91][40/66] loss=0.738734 avg=0.845020 VRAM=38.6GiB | 30.5% done | ETA(epoch): 499s + [Epoch 91][50/66] loss=0.822293 avg=0.845822 VRAM=38.7GiB | 30.6% done | ETA(epoch): 307s + [Epoch 91][60/66] loss=1.031441 avg=0.846514 VRAM=38.6GiB | 30.6% done | ETA(epoch): 115s + Train loss: 0.850543 (1267.5s) ETA: 4887min + Val loss: 0.871515 [t_0.0-0.2=1.0796 t_0.2-0.4=0.9921 t_0.4-0.6=0.8758 t_0.6-0.8=0.6983 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 92/299 (31% done) --- + [Epoch 92][10/66] loss=0.806779 avg=0.889519 VRAM=38.7GiB | 30.7% done | ETA(epoch): 1073s + [Epoch 92][20/66] loss=0.890523 avg=0.863537 VRAM=38.6GiB | 30.8% done | ETA(epoch): 883s + [Epoch 92][30/66] loss=0.969335 avg=0.875155 VRAM=38.7GiB | 30.8% done | ETA(epoch): 691s + [Epoch 92][40/66] loss=0.940190 avg=0.868956 VRAM=38.6GiB | 30.9% done | ETA(epoch): 499s + [Epoch 92][50/66] loss=0.844275 avg=0.868155 VRAM=38.7GiB | 30.9% done | ETA(epoch): 307s + [Epoch 92][60/66] loss=0.785355 avg=0.870127 VRAM=38.6GiB | 31.0% done | ETA(epoch): 115s + Train loss: 0.869093 (1266.9s) ETA: 4862min + Val loss: 0.847903 [t_0.0-0.2=1.0851 t_0.2-0.4=0.9484 t_0.4-0.6=0.8440 t_0.6-0.8=0.6971 t_0.8-1.0=0.6523] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 93/299 (31% done) --- + [Epoch 93][10/66] loss=0.789242 avg=0.824367 VRAM=38.7GiB | 31.1% done | ETA(epoch): 1091s + [Epoch 93][20/66] loss=0.816142 avg=0.831229 VRAM=38.6GiB | 31.1% done | ETA(epoch): 890s + [Epoch 93][30/66] loss=0.879848 avg=0.840729 VRAM=38.7GiB | 31.2% done | ETA(epoch): 695s + [Epoch 93][40/66] loss=0.852486 avg=0.856810 VRAM=38.6GiB | 31.2% done | ETA(epoch): 501s + [Epoch 93][50/66] loss=0.791593 avg=0.856719 VRAM=38.7GiB | 31.3% done | ETA(epoch): 308s + [Epoch 93][60/66] loss=0.852023 avg=0.863209 VRAM=38.6GiB | 31.3% done | ETA(epoch): 116s + Train loss: 0.868233 (1270.4s) ETA: 4837min + Val loss: 0.851137 [t_0.0-0.2=1.0854 t_0.2-0.4=0.9769 t_0.4-0.6=0.8557 t_0.6-0.8=0.7228 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 94/299 (31% done) --- + [Epoch 94][10/66] loss=0.804330 avg=0.830798 VRAM=38.7GiB | 31.4% done | ETA(epoch): 1074s + [Epoch 94][20/66] loss=0.903578 avg=0.859064 VRAM=38.6GiB | 31.4% done | ETA(epoch): 882s + [Epoch 94][30/66] loss=0.796290 avg=0.860333 VRAM=38.7GiB | 31.5% done | ETA(epoch): 691s + [Epoch 94][40/66] loss=0.786012 avg=0.860699 VRAM=38.6GiB | 31.5% done | ETA(epoch): 499s + [Epoch 94][50/66] loss=0.808764 avg=0.861938 VRAM=38.7GiB | 31.6% done | ETA(epoch): 307s + [Epoch 94][60/66] loss=0.921456 avg=0.862108 VRAM=38.6GiB | 31.6% done | ETA(epoch): 115s + Train loss: 0.862396 (1268.3s) ETA: 4812min + Val loss: 0.842584 [t_0.0-0.2=1.0656 t_0.2-0.4=0.9975 t_0.4-0.6=0.8399 t_0.6-0.8=0.7268 t_0.8-1.0=0.6434] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 95/299 (32% done) --- + [Epoch 95][10/66] loss=0.829399 avg=0.832191 VRAM=38.7GiB | 31.7% done | ETA(epoch): 1074s + [Epoch 95][20/66] loss=0.936516 avg=0.856105 VRAM=38.6GiB | 31.8% done | ETA(epoch): 883s + [Epoch 95][30/66] loss=0.899032 avg=0.859344 VRAM=38.7GiB | 31.8% done | ETA(epoch): 691s + [Epoch 95][40/66] loss=0.872707 avg=0.854203 VRAM=38.6GiB | 31.9% done | ETA(epoch): 499s + [Epoch 95][50/66] loss=0.833289 avg=0.862313 VRAM=38.7GiB | 31.9% done | ETA(epoch): 307s + [Epoch 95][60/66] loss=0.846850 avg=0.864653 VRAM=38.6GiB | 32.0% done | ETA(epoch): 115s + Train loss: 0.864475 (1269.7s) ETA: 4787min + Val loss: 0.813897 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9463 t_0.4-0.6=0.8382 t_0.6-0.8=0.7115 t_0.8-1.0=0.6493] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0095 (BEST) + Deleted old checkpoint: checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 96/299 (32% done) --- + [Epoch 96][10/66] loss=0.781842 avg=0.846492 VRAM=38.7GiB | 32.1% done | ETA(epoch): 1077s + [Epoch 96][20/66] loss=0.801173 avg=0.860613 VRAM=38.6GiB | 32.1% done | ETA(epoch): 884s + [Epoch 96][30/66] loss=0.918636 avg=0.864435 VRAM=38.7GiB | 32.2% done | ETA(epoch): 691s + [Epoch 96][40/66] loss=0.849381 avg=0.854338 VRAM=38.6GiB | 32.2% done | ETA(epoch): 499s + [Epoch 96][50/66] loss=0.869656 avg=0.859899 VRAM=38.7GiB | 32.3% done | ETA(epoch): 313s + [Epoch 96][60/66] loss=0.911374 avg=0.861963 VRAM=38.6GiB | 32.3% done | ETA(epoch): 117s + Train loss: 0.863516 (1287.1s) ETA: 4762min + Val loss: 0.842300 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9889 t_0.4-0.6=0.8584 t_0.6-0.8=0.7164 t_0.8-1.0=0.6650] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 19.5/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 97/299 (32% done) --- + [Epoch 97][10/66] loss=0.937833 avg=0.887353 VRAM=38.7GiB | 32.4% done | ETA(epoch): 1075s + [Epoch 97][20/66] loss=0.860866 avg=0.873778 VRAM=38.6GiB | 32.4% done | ETA(epoch): 883s + [Epoch 97][30/66] loss=0.828533 avg=0.877767 VRAM=38.7GiB | 32.5% done | ETA(epoch): 691s + [Epoch 97][40/66] loss=0.942115 avg=0.865536 VRAM=38.6GiB | 32.5% done | ETA(epoch): 499s + [Epoch 97][50/66] loss=0.941356 avg=0.866024 VRAM=38.7GiB | 32.6% done | ETA(epoch): 307s + [Epoch 97][60/66] loss=0.837143 avg=0.860793 VRAM=38.6GiB | 32.6% done | ETA(epoch): 115s + Train loss: 0.864756 (1267.5s) ETA: 4737min + Val loss: 0.852580 [t_0.0-0.2=1.0766 t_0.2-0.4=1.0272 t_0.4-0.6=0.8612 t_0.6-0.8=0.7073 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 98/299 (33% done) --- + [Epoch 98][10/66] loss=0.987492 avg=0.849812 VRAM=38.7GiB | 32.7% done | ETA(epoch): 1076s + [Epoch 98][20/66] loss=0.766366 avg=0.849266 VRAM=38.6GiB | 32.8% done | ETA(epoch): 884s + [Epoch 98][30/66] loss=0.989442 avg=0.855450 VRAM=38.7GiB | 32.8% done | ETA(epoch): 694s + [Epoch 98][40/66] loss=0.851608 avg=0.861278 VRAM=38.6GiB | 32.9% done | ETA(epoch): 501s + [Epoch 98][50/66] loss=0.807638 avg=0.858684 VRAM=38.7GiB | 32.9% done | ETA(epoch): 308s + [Epoch 98][60/66] loss=0.867773 avg=0.857908 VRAM=38.6GiB | 33.0% done | ETA(epoch): 115s + Train loss: 0.861672 (1270.0s) ETA: 4712min + Val loss: 0.844169 [t_0.0-0.2=1.0637 t_0.2-0.4=0.9745 t_0.4-0.6=0.8770 t_0.6-0.8=0.7276 t_0.8-1.0=0.6594] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0098 +[MEM @ epoch 98 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 99/299 (33% done) --- + [Epoch 99][10/66] loss=0.679792 avg=0.865722 VRAM=38.7GiB | 33.1% done | ETA(epoch): 1075s + [Epoch 99][20/66] loss=0.749062 avg=0.861549 VRAM=38.6GiB | 33.1% done | ETA(epoch): 883s + [Epoch 99][30/66] loss=0.835393 avg=0.860883 VRAM=38.7GiB | 33.2% done | ETA(epoch): 691s + [Epoch 99][40/66] loss=0.850963 avg=0.853638 VRAM=38.6GiB | 33.2% done | ETA(epoch): 499s + [Epoch 99][50/66] loss=0.830811 avg=0.856753 VRAM=38.7GiB | 33.3% done | ETA(epoch): 307s + [Epoch 99][60/66] loss=0.783823 avg=0.850992 VRAM=38.6GiB | 33.3% done | ETA(epoch): 115s + Train loss: 0.848351 (1267.2s) ETA: 4687min + Val loss: 0.881852 [t_0.0-0.2=1.0882 t_0.2-0.4=1.0172 t_0.4-0.6=0.8131 t_0.6-0.8=0.7268 t_0.8-1.0=0.6552] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 100/299 (33% done) --- + [Epoch 100][10/66] loss=0.707337 avg=0.826334 VRAM=38.7GiB | 33.4% done | ETA(epoch): 1093s + [Epoch 100][20/66] loss=0.862292 avg=0.852078 VRAM=38.6GiB | 33.4% done | ETA(epoch): 891s + [Epoch 100][30/66] loss=0.849977 avg=0.857029 VRAM=38.7GiB | 33.5% done | ETA(epoch): 695s + [Epoch 100][40/66] loss=0.758742 avg=0.868305 VRAM=38.6GiB | 33.5% done | ETA(epoch): 501s + [Epoch 100][50/66] loss=0.937653 avg=0.872226 VRAM=38.7GiB | 33.6% done | ETA(epoch): 308s + [Epoch 100][60/66] loss=1.010351 avg=0.876112 VRAM=38.6GiB | 33.6% done | ETA(epoch): 116s + Train loss: 0.876511 (1270.6s) ETA: 4663min + Val loss: 0.841915 [t_0.0-0.2=1.0608 t_0.2-0.4=1.0381 t_0.4-0.6=0.8698 t_0.6-0.8=0.7129 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 101/299 (34% done) --- + [Epoch 101][10/66] loss=0.798365 avg=0.862447 VRAM=38.7GiB | 33.7% done | ETA(epoch): 1076s + [Epoch 101][20/66] loss=0.764806 avg=0.860402 VRAM=38.6GiB | 33.8% done | ETA(epoch): 883s + [Epoch 101][30/66] loss=0.932255 avg=0.870261 VRAM=38.7GiB | 33.8% done | ETA(epoch): 691s + [Epoch 101][40/66] loss=0.901447 avg=0.865949 VRAM=38.6GiB | 33.9% done | ETA(epoch): 499s + [Epoch 101][50/66] loss=0.899411 avg=0.862845 VRAM=38.7GiB | 33.9% done | ETA(epoch): 307s + [Epoch 101][60/66] loss=0.851224 avg=0.855968 VRAM=38.6GiB | 34.0% done | ETA(epoch): 115s + Train loss: 0.857470 (1267.2s) ETA: 4638min + Val loss: 0.859045 [t_0.0-0.2=1.0919 t_0.2-0.4=0.9764 t_0.4-0.6=0.8484 t_0.6-0.8=0.7311 t_0.8-1.0=0.6531] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 102/299 (34% done) --- + [Epoch 102][10/66] loss=0.819327 avg=0.858897 VRAM=38.7GiB | 34.1% done | ETA(epoch): 1076s + [Epoch 102][20/66] loss=0.890353 avg=0.839417 VRAM=38.6GiB | 34.1% done | ETA(epoch): 883s + [Epoch 102][30/66] loss=0.852891 avg=0.844558 VRAM=38.7GiB | 34.2% done | ETA(epoch): 691s + [Epoch 102][40/66] loss=0.998418 avg=0.851381 VRAM=38.6GiB | 34.2% done | ETA(epoch): 499s + [Epoch 102][50/66] loss=0.906683 avg=0.855532 VRAM=38.7GiB | 34.3% done | ETA(epoch): 307s + [Epoch 102][60/66] loss=0.831913 avg=0.862141 VRAM=38.6GiB | 34.3% done | ETA(epoch): 115s + Train loss: 0.863573 (1267.3s) ETA: 4613min + Val loss: 0.880758 [t_0.0-0.2=1.0725 t_0.2-0.4=1.0106 t_0.4-0.6=0.8408 t_0.6-0.8=0.7128 t_0.8-1.0=0.6817] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 103/299 (34% done) --- + [Epoch 103][10/66] loss=0.768569 avg=0.888777 VRAM=38.7GiB | 34.4% done | ETA(epoch): 1075s + [Epoch 103][20/66] loss=0.776707 avg=0.862823 VRAM=38.6GiB | 34.4% done | ETA(epoch): 883s + [Epoch 103][30/66] loss=0.860144 avg=0.869215 VRAM=38.7GiB | 34.5% done | ETA(epoch): 691s + [Epoch 103][40/66] loss=0.872193 avg=0.863494 VRAM=38.6GiB | 34.5% done | ETA(epoch): 499s + [Epoch 103][50/66] loss=0.781946 avg=0.858345 VRAM=38.7GiB | 34.6% done | ETA(epoch): 307s + [Epoch 103][60/66] loss=0.926681 avg=0.858074 VRAM=38.6GiB | 34.6% done | ETA(epoch): 115s + Train loss: 0.855992 (1267.1s) ETA: 4588min + Val loss: 0.841009 [t_0.0-0.2=1.0758 t_0.2-0.4=1.0272 t_0.4-0.6=0.8502 t_0.6-0.8=0.6756 t_0.8-1.0=0.6527] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 104/299 (35% done) --- + [Epoch 104][10/66] loss=0.855486 avg=0.851349 VRAM=38.7GiB | 34.7% done | ETA(epoch): 1076s + [Epoch 104][20/66] loss=0.793957 avg=0.845954 VRAM=38.6GiB | 34.8% done | ETA(epoch): 882s + [Epoch 104][30/66] loss=0.912438 avg=0.845964 VRAM=38.7GiB | 34.8% done | ETA(epoch): 691s + [Epoch 104][40/66] loss=0.754521 avg=0.865309 VRAM=38.6GiB | 34.9% done | ETA(epoch): 500s + [Epoch 104][50/66] loss=0.872342 avg=0.860091 VRAM=38.7GiB | 34.9% done | ETA(epoch): 308s + [Epoch 104][60/66] loss=0.742773 avg=0.860288 VRAM=38.6GiB | 35.0% done | ETA(epoch): 115s + Train loss: 0.857122 (1268.8s) ETA: 4564min + Val loss: 0.811569 [t_0.0-0.2=1.0744 t_0.2-0.4=0.9887 t_0.4-0.6=0.8204 t_0.6-0.8=0.7182 t_0.8-1.0=0.6501] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0104 (BEST) + Deleted old checkpoint: checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 105/299 (35% done) --- + [Epoch 105][10/66] loss=0.842854 avg=0.875085 VRAM=38.7GiB | 35.1% done | ETA(epoch): 1074s + [Epoch 105][20/66] loss=1.061811 avg=0.881139 VRAM=38.6GiB | 35.1% done | ETA(epoch): 882s + [Epoch 105][30/66] loss=0.828484 avg=0.865721 VRAM=38.7GiB | 35.2% done | ETA(epoch): 690s + [Epoch 105][40/66] loss=1.016454 avg=0.877483 VRAM=38.6GiB | 35.2% done | ETA(epoch): 510s + [Epoch 105][50/66] loss=0.931515 avg=0.868579 VRAM=38.7GiB | 35.3% done | ETA(epoch): 313s + [Epoch 105][60/66] loss=0.791349 avg=0.865732 VRAM=38.6GiB | 35.3% done | ETA(epoch): 117s + Train loss: 0.867332 (1284.6s) ETA: 4539min + Val loss: 0.842561 [t_0.0-0.2=1.0842 t_0.2-0.4=0.9826 t_0.4-0.6=0.8516 t_0.6-0.8=0.6968 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 106/299 (35% done) --- + [Epoch 106][10/66] loss=0.822894 avg=0.841237 VRAM=38.7GiB | 35.4% done | ETA(epoch): 1074s + [Epoch 106][20/66] loss=0.889432 avg=0.863572 VRAM=38.6GiB | 35.4% done | ETA(epoch): 883s + [Epoch 106][30/66] loss=0.915390 avg=0.864090 VRAM=38.7GiB | 35.5% done | ETA(epoch): 691s + [Epoch 106][40/66] loss=0.815298 avg=0.859378 VRAM=38.6GiB | 35.5% done | ETA(epoch): 499s + [Epoch 106][50/66] loss=0.931338 avg=0.856359 VRAM=38.7GiB | 35.6% done | ETA(epoch): 307s + [Epoch 106][60/66] loss=0.977590 avg=0.867957 VRAM=38.6GiB | 35.6% done | ETA(epoch): 115s + Train loss: 0.868391 (1267.3s) ETA: 4515min + Val loss: 0.859613 [t_0.0-0.2=1.0619 t_0.2-0.4=1.0085 t_0.4-0.6=0.8219 t_0.6-0.8=0.7306 t_0.8-1.0=0.6342] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 107/299 (36% done) --- + [Epoch 107][10/66] loss=1.023521 avg=0.876832 VRAM=38.7GiB | 35.7% done | ETA(epoch): 1075s + [Epoch 107][20/66] loss=0.784307 avg=0.857386 VRAM=38.6GiB | 35.8% done | ETA(epoch): 892s + [Epoch 107][30/66] loss=0.815456 avg=0.850512 VRAM=38.7GiB | 35.8% done | ETA(epoch): 696s + [Epoch 107][40/66] loss=0.769345 avg=0.862148 VRAM=38.6GiB | 35.9% done | ETA(epoch): 501s + [Epoch 107][50/66] loss=0.978483 avg=0.863074 VRAM=38.7GiB | 35.9% done | ETA(epoch): 308s + [Epoch 107][60/66] loss=0.892956 avg=0.863468 VRAM=38.6GiB | 36.0% done | ETA(epoch): 116s + Train loss: 0.865041 (1270.8s) ETA: 4490min + Val loss: 0.883903 [t_0.0-0.2=1.0744 t_0.2-0.4=1.0134 t_0.4-0.6=0.8598 t_0.6-0.8=0.7127 t_0.8-1.0=0.6429] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0107 +[MEM @ epoch 107 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 108/299 (36% done) --- + [Epoch 108][10/66] loss=0.918885 avg=0.873589 VRAM=38.7GiB | 36.1% done | ETA(epoch): 1087s + [Epoch 108][20/66] loss=0.971575 avg=0.862435 VRAM=38.6GiB | 36.1% done | ETA(epoch): 888s + [Epoch 108][30/66] loss=0.756556 avg=0.853152 VRAM=38.7GiB | 36.2% done | ETA(epoch): 694s + [Epoch 108][40/66] loss=0.955186 avg=0.861690 VRAM=38.6GiB | 36.2% done | ETA(epoch): 501s + [Epoch 108][50/66] loss=0.797275 avg=0.861806 VRAM=38.7GiB | 36.3% done | ETA(epoch): 308s + [Epoch 108][60/66] loss=0.859753 avg=0.861895 VRAM=38.6GiB | 36.3% done | ETA(epoch): 116s + Train loss: 0.860716 (1270.5s) ETA: 4466min + Val loss: 0.872801 [t_0.0-0.2=1.0830 t_0.2-0.4=1.0071 t_0.4-0.6=0.8353 t_0.6-0.8=0.7086 t_0.8-1.0=0.6685] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 109/299 (36% done) --- + [Epoch 109][10/66] loss=1.020916 avg=0.902935 VRAM=38.7GiB | 36.4% done | ETA(epoch): 1074s + [Epoch 109][20/66] loss=0.856909 avg=0.883469 VRAM=38.6GiB | 36.4% done | ETA(epoch): 883s + [Epoch 109][30/66] loss=0.802624 avg=0.875493 VRAM=38.7GiB | 36.5% done | ETA(epoch): 691s + [Epoch 109][40/66] loss=0.834306 avg=0.866127 VRAM=38.6GiB | 36.5% done | ETA(epoch): 499s + [Epoch 109][50/66] loss=0.896399 avg=0.868294 VRAM=38.7GiB | 36.6% done | ETA(epoch): 307s + [Epoch 109][60/66] loss=0.908138 avg=0.876225 VRAM=38.6GiB | 36.6% done | ETA(epoch): 115s + Train loss: 0.877800 (1272.3s) ETA: 4441min + Val loss: 0.861098 [t_0.0-0.2=1.0552 t_0.2-0.4=0.9945 t_0.4-0.6=0.8486 t_0.6-0.8=0.7160 t_0.8-1.0=0.6588] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 110/299 (37% done) --- + [Epoch 110][10/66] loss=0.813677 avg=0.836899 VRAM=38.7GiB | 36.7% done | ETA(epoch): 1074s + [Epoch 110][20/66] loss=0.848039 avg=0.817130 VRAM=38.6GiB | 36.8% done | ETA(epoch): 883s + [Epoch 110][30/66] loss=0.775279 avg=0.838072 VRAM=38.7GiB | 36.8% done | ETA(epoch): 691s + [Epoch 110][40/66] loss=0.952471 avg=0.845673 VRAM=38.6GiB | 36.9% done | ETA(epoch): 499s + [Epoch 110][50/66] loss=0.918807 avg=0.852464 VRAM=38.7GiB | 36.9% done | ETA(epoch): 307s + [Epoch 110][60/66] loss=0.776143 avg=0.850361 VRAM=38.6GiB | 37.0% done | ETA(epoch): 115s + Train loss: 0.852819 (1269.7s) ETA: 4417min + Val loss: 0.878836 [t_0.0-0.2=1.0742 t_0.2-0.4=1.0237 t_0.4-0.6=0.8422 t_0.6-0.8=0.7434 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 111/299 (37% done) --- + [Epoch 111][10/66] loss=0.769104 avg=0.867349 VRAM=38.7GiB | 37.1% done | ETA(epoch): 1075s + [Epoch 111][20/66] loss=0.859824 avg=0.881804 VRAM=38.6GiB | 37.1% done | ETA(epoch): 883s + [Epoch 111][30/66] loss=0.702154 avg=0.873571 VRAM=38.7GiB | 37.2% done | ETA(epoch): 691s + [Epoch 111][40/66] loss=0.895654 avg=0.869507 VRAM=38.6GiB | 37.2% done | ETA(epoch): 499s + [Epoch 111][50/66] loss=0.835729 avg=0.863139 VRAM=38.7GiB | 37.3% done | ETA(epoch): 307s + [Epoch 111][60/66] loss=0.909972 avg=0.857175 VRAM=38.6GiB | 37.3% done | ETA(epoch): 115s + Train loss: 0.858032 (1267.9s) ETA: 4392min + Val loss: 0.824633 [t_0.0-0.2=1.0931 t_0.2-0.4=1.0037 t_0.4-0.6=0.8272 t_0.6-0.8=0.7162 t_0.8-1.0=0.6469] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 112/299 (37% done) --- + [Epoch 112][10/66] loss=0.725716 avg=0.820207 VRAM=38.7GiB | 37.4% done | ETA(epoch): 1075s + [Epoch 112][20/66] loss=0.820025 avg=0.836291 VRAM=38.6GiB | 37.4% done | ETA(epoch): 884s + [Epoch 112][30/66] loss=0.800888 avg=0.853900 VRAM=38.7GiB | 37.5% done | ETA(epoch): 692s + [Epoch 112][40/66] loss=0.771399 avg=0.863454 VRAM=38.6GiB | 37.5% done | ETA(epoch): 501s + [Epoch 112][50/66] loss=0.836189 avg=0.866768 VRAM=38.7GiB | 37.6% done | ETA(epoch): 308s + [Epoch 112][60/66] loss=0.816546 avg=0.866800 VRAM=38.6GiB | 37.6% done | ETA(epoch): 115s + Train loss: 0.870582 (1269.9s) ETA: 4368min + Val loss: 0.852974 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0025 t_0.4-0.6=0.8487 t_0.6-0.8=0.6887 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 113/299 (38% done) --- + [Epoch 113][10/66] loss=0.918006 avg=0.865943 VRAM=38.7GiB | 37.7% done | ETA(epoch): 1074s + [Epoch 113][20/66] loss=0.813233 avg=0.859364 VRAM=38.6GiB | 37.8% done | ETA(epoch): 882s + [Epoch 113][30/66] loss=0.933331 avg=0.860306 VRAM=38.7GiB | 37.8% done | ETA(epoch): 691s + [Epoch 113][40/66] loss=0.834739 avg=0.867955 VRAM=38.6GiB | 37.9% done | ETA(epoch): 499s + [Epoch 113][50/66] loss=0.836336 avg=0.872393 VRAM=38.7GiB | 37.9% done | ETA(epoch): 307s + [Epoch 113][60/66] loss=0.839982 avg=0.861995 VRAM=38.6GiB | 38.0% done | ETA(epoch): 115s + Train loss: 0.862170 (1267.5s) ETA: 4344min + Val loss: 0.847482 [t_0.0-0.2=1.0785 t_0.2-0.4=0.9851 t_0.4-0.6=0.8657 t_0.6-0.8=0.7265 t_0.8-1.0=0.6659] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 114/299 (38% done) --- + [Epoch 114][10/66] loss=0.872971 avg=0.853259 VRAM=38.7GiB | 38.1% done | ETA(epoch): 1075s + [Epoch 114][20/66] loss=0.770475 avg=0.851473 VRAM=38.6GiB | 38.1% done | ETA(epoch): 883s + [Epoch 114][30/66] loss=0.904781 avg=0.856303 VRAM=38.7GiB | 38.2% done | ETA(epoch): 692s + [Epoch 114][40/66] loss=0.701344 avg=0.849344 VRAM=38.6GiB | 38.2% done | ETA(epoch): 499s + [Epoch 114][50/66] loss=0.794602 avg=0.850215 VRAM=38.7GiB | 38.3% done | ETA(epoch): 307s + [Epoch 114][60/66] loss=0.811940 avg=0.846805 VRAM=38.6GiB | 38.3% done | ETA(epoch): 115s + Train loss: 0.850057 (1268.2s) ETA: 4319min + Val loss: 0.891056 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0165 t_0.4-0.6=0.8172 t_0.6-0.8=0.7369 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 115/299 (38% done) --- + [Epoch 115][10/66] loss=0.948415 avg=0.873687 VRAM=38.7GiB | 38.4% done | ETA(epoch): 1077s + [Epoch 115][20/66] loss=0.775373 avg=0.847412 VRAM=38.6GiB | 38.4% done | ETA(epoch): 885s + [Epoch 115][30/66] loss=0.774699 avg=0.850346 VRAM=38.7GiB | 38.5% done | ETA(epoch): 693s + [Epoch 115][40/66] loss=0.905183 avg=0.854816 VRAM=38.6GiB | 38.5% done | ETA(epoch): 500s + [Epoch 115][50/66] loss=0.849527 avg=0.858814 VRAM=38.7GiB | 38.6% done | ETA(epoch): 308s + [Epoch 115][60/66] loss=0.819797 avg=0.853045 VRAM=38.6GiB | 38.6% done | ETA(epoch): 115s + Train loss: 0.851671 (1269.2s) ETA: 4295min + Val loss: 0.870576 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0050 t_0.4-0.6=0.8622 t_0.6-0.8=0.7048 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 116/299 (39% done) --- + [Epoch 116][10/66] loss=0.910492 avg=0.862083 VRAM=38.7GiB | 38.7% done | ETA(epoch): 1097s + [Epoch 116][20/66] loss=0.953223 avg=0.850827 VRAM=38.6GiB | 38.8% done | ETA(epoch): 892s + [Epoch 116][30/66] loss=0.905785 avg=0.847631 VRAM=38.7GiB | 38.8% done | ETA(epoch): 696s + [Epoch 116][40/66] loss=0.830655 avg=0.848074 VRAM=38.6GiB | 38.9% done | ETA(epoch): 502s + [Epoch 116][50/66] loss=0.814507 avg=0.845591 VRAM=38.7GiB | 38.9% done | ETA(epoch): 309s + [Epoch 116][60/66] loss=0.814340 avg=0.839113 VRAM=38.6GiB | 39.0% done | ETA(epoch): 116s + Train loss: 0.839892 (1271.8s) ETA: 4271min + Val loss: 0.848096 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9990 t_0.4-0.6=0.8221 t_0.6-0.8=0.7184 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 117/299 (39% done) --- + [Epoch 117][10/66] loss=0.991918 avg=0.898374 VRAM=38.7GiB | 39.1% done | ETA(epoch): 1075s + [Epoch 117][20/66] loss=0.897056 avg=0.893629 VRAM=38.6GiB | 39.1% done | ETA(epoch): 884s + [Epoch 117][30/66] loss=0.896946 avg=0.887703 VRAM=38.7GiB | 39.2% done | ETA(epoch): 692s + [Epoch 117][40/66] loss=0.817093 avg=0.882021 VRAM=38.6GiB | 39.2% done | ETA(epoch): 500s + [Epoch 117][50/66] loss=0.850016 avg=0.874241 VRAM=38.7GiB | 39.3% done | ETA(epoch): 308s + [Epoch 117][60/66] loss=0.845396 avg=0.870545 VRAM=38.6GiB | 39.3% done | ETA(epoch): 116s + Train loss: 0.868508 (1272.4s) ETA: 4247min + Val loss: 0.867382 [t_0.0-0.2=1.0701 t_0.2-0.4=0.9988 t_0.4-0.6=0.7830 t_0.6-0.8=0.7305 t_0.8-1.0=0.6518] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 118/299 (39% done) --- + [Epoch 118][10/66] loss=0.957646 avg=0.859526 VRAM=38.7GiB | 39.4% done | ETA(epoch): 1075s + [Epoch 118][20/66] loss=0.812602 avg=0.850654 VRAM=38.6GiB | 39.4% done | ETA(epoch): 883s + [Epoch 118][30/66] loss=0.721708 avg=0.835873 VRAM=38.7GiB | 39.5% done | ETA(epoch): 691s + [Epoch 118][40/66] loss=0.948969 avg=0.848118 VRAM=38.6GiB | 39.5% done | ETA(epoch): 499s + [Epoch 118][50/66] loss=0.835788 avg=0.844155 VRAM=38.7GiB | 39.6% done | ETA(epoch): 307s + [Epoch 118][60/66] loss=0.974827 avg=0.844750 VRAM=38.6GiB | 39.6% done | ETA(epoch): 115s + Train loss: 0.849348 (1268.1s) ETA: 4222min + Val loss: 0.871310 [t_0.0-0.2=1.0559 t_0.2-0.4=1.0172 t_0.4-0.6=0.8748 t_0.6-0.8=0.7037 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 119/299 (40% done) --- + [Epoch 119][10/66] loss=0.844312 avg=0.877483 VRAM=38.7GiB | 39.7% done | ETA(epoch): 1075s + [Epoch 119][20/66] loss=0.884663 avg=0.875649 VRAM=38.6GiB | 39.8% done | ETA(epoch): 883s + [Epoch 119][30/66] loss=0.886157 avg=0.868869 VRAM=38.7GiB | 39.8% done | ETA(epoch): 692s + [Epoch 119][40/66] loss=0.849521 avg=0.873225 VRAM=38.6GiB | 39.9% done | ETA(epoch): 499s + [Epoch 119][50/66] loss=0.845381 avg=0.869671 VRAM=38.7GiB | 39.9% done | ETA(epoch): 308s + [Epoch 119][60/66] loss=0.901835 avg=0.872608 VRAM=38.6GiB | 40.0% done | ETA(epoch): 115s + Train loss: 0.866268 (1269.8s) ETA: 4198min + Val loss: 0.847478 [t_0.0-0.2=1.0732 t_0.2-0.4=1.0020 t_0.4-0.6=0.8283 t_0.6-0.8=0.7155 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 120/299 (40% done) --- + [MilestoneVis] train_0 step 7920 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 7920 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 7920 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 7920 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 7920 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 7920 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 7920 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 7920 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 7920 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 7920 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 120][10/66] loss=0.972220 avg=0.855851 VRAM=38.7GiB | 40.1% done | ETA(epoch): 6849s + [Epoch 120][20/66] loss=0.842627 avg=0.852267 VRAM=38.6GiB | 40.1% done | ETA(epoch): 3254s + [Epoch 120][30/66] loss=0.908397 avg=0.851938 VRAM=38.7GiB | 40.2% done | ETA(epoch): 1928s + [Epoch 120][40/66] loss=0.994705 avg=0.848798 VRAM=38.6GiB | 40.2% done | ETA(epoch): 1169s + [Epoch 120][50/66] loss=0.875831 avg=0.851590 VRAM=38.7GiB | 40.3% done | ETA(epoch): 642s + [Epoch 120][60/66] loss=0.783125 avg=0.854045 VRAM=38.6GiB | 40.3% done | ETA(epoch): 220s + Train loss: 0.853371 (2314.6s) ETA: 4200min + Val loss: 0.852036 [t_0.0-0.2=1.0500 t_0.2-0.4=1.0198 t_0.4-0.6=0.8632 t_0.6-0.8=0.7150 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 121/299 (40% done) --- + [Epoch 121][10/66] loss=0.896852 avg=0.899315 VRAM=38.7GiB | 40.4% done | ETA(epoch): 1073s + [Epoch 121][20/66] loss=0.865983 avg=0.863900 VRAM=38.6GiB | 40.4% done | ETA(epoch): 882s + [Epoch 121][30/66] loss=0.926843 avg=0.866099 VRAM=38.7GiB | 40.5% done | ETA(epoch): 690s + [Epoch 121][40/66] loss=0.999985 avg=0.868595 VRAM=38.6GiB | 40.5% done | ETA(epoch): 499s + [Epoch 121][50/66] loss=0.913925 avg=0.869715 VRAM=38.7GiB | 40.6% done | ETA(epoch): 307s + [Epoch 121][60/66] loss=0.862345 avg=0.864567 VRAM=38.6GiB | 40.6% done | ETA(epoch): 115s + Train loss: 0.862729 (1266.1s) ETA: 4175min + Val loss: 0.845551 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9809 t_0.4-0.6=0.8506 t_0.6-0.8=0.7185 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 122/299 (41% done) --- + [Epoch 122][10/66] loss=0.770397 avg=0.844221 VRAM=38.7GiB | 40.7% done | ETA(epoch): 1074s + [Epoch 122][20/66] loss=0.963255 avg=0.859415 VRAM=38.6GiB | 40.8% done | ETA(epoch): 883s + [Epoch 122][30/66] loss=0.819885 avg=0.849183 VRAM=38.7GiB | 40.8% done | ETA(epoch): 694s + [Epoch 122][40/66] loss=0.890258 avg=0.847065 VRAM=38.6GiB | 40.9% done | ETA(epoch): 500s + [Epoch 122][50/66] loss=0.719200 avg=0.849522 VRAM=38.7GiB | 40.9% done | ETA(epoch): 308s + [Epoch 122][60/66] loss=0.906860 avg=0.855196 VRAM=38.6GiB | 41.0% done | ETA(epoch): 115s + Train loss: 0.853975 (1269.1s) ETA: 4151min + Val loss: 0.855912 [t_0.0-0.2=1.0605 t_0.2-0.4=0.9940 t_0.4-0.6=0.8428 t_0.6-0.8=0.7203 t_0.8-1.0=0.6593] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 123/299 (41% done) --- + [Epoch 123][10/66] loss=0.804983 avg=0.850791 VRAM=38.7GiB | 41.1% done | ETA(epoch): 1072s + [Epoch 123][20/66] loss=0.894884 avg=0.866705 VRAM=38.6GiB | 41.1% done | ETA(epoch): 886s + [Epoch 123][30/66] loss=0.758751 avg=0.860466 VRAM=38.7GiB | 41.2% done | ETA(epoch): 692s + [Epoch 123][40/66] loss=1.029475 avg=0.863322 VRAM=38.6GiB | 41.2% done | ETA(epoch): 500s + [Epoch 123][50/66] loss=0.937271 avg=0.860312 VRAM=38.7GiB | 41.3% done | ETA(epoch): 307s + [Epoch 123][60/66] loss=0.903350 avg=0.858498 VRAM=38.6GiB | 41.3% done | ETA(epoch): 115s + Train loss: 0.859069 (1267.9s) ETA: 4126min + Val loss: 0.855692 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9554 t_0.4-0.6=0.8566 t_0.6-0.8=0.7194 t_0.8-1.0=0.6494] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 124/299 (41% done) --- + [Epoch 124][10/66] loss=0.795019 avg=0.875205 VRAM=38.7GiB | 41.4% done | ETA(epoch): 1074s + [Epoch 124][20/66] loss=0.771340 avg=0.830329 VRAM=38.6GiB | 41.4% done | ETA(epoch): 882s + [Epoch 124][30/66] loss=0.684459 avg=0.830977 VRAM=38.7GiB | 41.5% done | ETA(epoch): 690s + [Epoch 124][40/66] loss=0.755129 avg=0.839055 VRAM=38.6GiB | 41.5% done | ETA(epoch): 499s + [Epoch 124][50/66] loss=0.851861 avg=0.841508 VRAM=38.7GiB | 41.6% done | ETA(epoch): 307s + [Epoch 124][60/66] loss=0.876930 avg=0.845649 VRAM=38.6GiB | 41.6% done | ETA(epoch): 115s + Train loss: 0.844983 (1265.8s) ETA: 4102min + Val loss: 0.865209 [t_0.0-0.2=1.0855 t_0.2-0.4=0.9790 t_0.4-0.6=0.8515 t_0.6-0.8=0.7197 t_0.8-1.0=0.6635] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 125/299 (42% done) --- + [Epoch 125][10/66] loss=0.987453 avg=0.866674 VRAM=38.7GiB | 41.7% done | ETA(epoch): 1087s + [Epoch 125][20/66] loss=0.914185 avg=0.852200 VRAM=38.6GiB | 41.8% done | ETA(epoch): 888s + [Epoch 125][30/66] loss=0.928623 avg=0.870312 VRAM=38.7GiB | 41.8% done | ETA(epoch): 694s + [Epoch 125][40/66] loss=0.827870 avg=0.864755 VRAM=38.6GiB | 41.9% done | ETA(epoch): 501s + [Epoch 125][50/66] loss=0.901121 avg=0.858949 VRAM=38.7GiB | 41.9% done | ETA(epoch): 308s + [Epoch 125][60/66] loss=0.754500 avg=0.858430 VRAM=38.6GiB | 42.0% done | ETA(epoch): 115s + Train loss: 0.853595 (1271.3s) ETA: 4077min + Val loss: 0.830618 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9507 t_0.4-0.6=0.8335 t_0.6-0.8=0.7323 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 126/299 (42% done) --- + [Epoch 126][10/66] loss=0.985721 avg=0.838734 VRAM=38.7GiB | 42.1% done | ETA(epoch): 1075s + [Epoch 126][20/66] loss=0.990719 avg=0.842516 VRAM=38.6GiB | 42.1% done | ETA(epoch): 883s + [Epoch 126][30/66] loss=0.813207 avg=0.835771 VRAM=38.7GiB | 42.2% done | ETA(epoch): 691s + [Epoch 126][40/66] loss=0.774008 avg=0.848560 VRAM=38.6GiB | 42.2% done | ETA(epoch): 499s + [Epoch 126][50/66] loss=0.854142 avg=0.854970 VRAM=38.7GiB | 42.3% done | ETA(epoch): 307s + [Epoch 126][60/66] loss=0.757393 avg=0.851503 VRAM=38.6GiB | 42.3% done | ETA(epoch): 115s + Train loss: 0.854778 (1268.1s) ETA: 4053min + Val loss: 0.813519 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9818 t_0.4-0.6=0.8292 t_0.6-0.8=0.7071 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 127/299 (42% done) --- + [Epoch 127][10/66] loss=0.963557 avg=0.877266 VRAM=38.7GiB | 42.4% done | ETA(epoch): 1074s + [Epoch 127][20/66] loss=0.887115 avg=0.859309 VRAM=38.6GiB | 42.4% done | ETA(epoch): 882s + [Epoch 127][30/66] loss=0.947664 avg=0.866036 VRAM=38.7GiB | 42.5% done | ETA(epoch): 691s + [Epoch 127][40/66] loss=0.785834 avg=0.857445 VRAM=38.6GiB | 42.5% done | ETA(epoch): 499s + [Epoch 127][50/66] loss=0.828912 avg=0.856188 VRAM=38.7GiB | 42.6% done | ETA(epoch): 309s + [Epoch 127][60/66] loss=0.839030 avg=0.852208 VRAM=38.6GiB | 42.6% done | ETA(epoch): 116s + Train loss: 0.854927 (1271.6s) ETA: 4028min + Val loss: 0.857016 [t_0.0-0.2=1.0761 t_0.2-0.4=1.0175 t_0.4-0.6=0.8519 t_0.6-0.8=0.7159 t_0.8-1.0=0.6498] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 128/299 (43% done) --- + [Epoch 128][10/66] loss=0.807672 avg=0.836722 VRAM=38.7GiB | 42.7% done | ETA(epoch): 1076s + [Epoch 128][20/66] loss=0.836284 avg=0.855325 VRAM=38.6GiB | 42.8% done | ETA(epoch): 883s + [Epoch 128][30/66] loss=0.826950 avg=0.848386 VRAM=38.7GiB | 42.8% done | ETA(epoch): 691s + [Epoch 128][40/66] loss=0.920138 avg=0.851954 VRAM=38.6GiB | 42.9% done | ETA(epoch): 499s + [Epoch 128][50/66] loss=0.756851 avg=0.854241 VRAM=38.7GiB | 42.9% done | ETA(epoch): 307s + [Epoch 128][60/66] loss=0.864309 avg=0.853280 VRAM=38.6GiB | 43.0% done | ETA(epoch): 115s + Train loss: 0.850907 (1267.0s) ETA: 4004min + Val loss: 0.848392 [t_0.0-0.2=1.0518 t_0.2-0.4=1.0008 t_0.4-0.6=0.8365 t_0.6-0.8=0.7270 t_0.8-1.0=0.6523] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 129/299 (43% done) --- + [Epoch 129][10/66] loss=0.894170 avg=0.830572 VRAM=38.7GiB | 43.1% done | ETA(epoch): 1075s + [Epoch 129][20/66] loss=0.768513 avg=0.834303 VRAM=38.6GiB | 43.1% done | ETA(epoch): 883s + [Epoch 129][30/66] loss=0.837907 avg=0.836937 VRAM=38.7GiB | 43.2% done | ETA(epoch): 692s + [Epoch 129][40/66] loss=0.865709 avg=0.841577 VRAM=38.6GiB | 43.2% done | ETA(epoch): 501s + [Epoch 129][50/66] loss=0.863172 avg=0.848108 VRAM=38.7GiB | 43.3% done | ETA(epoch): 308s + [Epoch 129][60/66] loss=0.809796 avg=0.841594 VRAM=38.6GiB | 43.3% done | ETA(epoch): 116s + Train loss: 0.841696 (1270.3s) ETA: 3980min + Val loss: 0.906046 [t_0.0-0.2=1.0720 t_0.2-0.4=0.9977 t_0.4-0.6=0.8841 t_0.6-0.8=0.6905 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 130/299 (43% done) --- + [Epoch 130][10/66] loss=0.863670 avg=0.875507 VRAM=38.7GiB | 43.4% done | ETA(epoch): 1076s + [Epoch 130][20/66] loss=0.798763 avg=0.884654 VRAM=38.6GiB | 43.4% done | ETA(epoch): 883s + [Epoch 130][30/66] loss=0.726157 avg=0.878169 VRAM=38.7GiB | 43.5% done | ETA(epoch): 697s + [Epoch 130][40/66] loss=0.876514 avg=0.876855 VRAM=38.6GiB | 43.5% done | ETA(epoch): 503s + [Epoch 130][50/66] loss=0.844097 avg=0.872824 VRAM=38.7GiB | 43.6% done | ETA(epoch): 309s + [Epoch 130][60/66] loss=0.956206 avg=0.868426 VRAM=38.6GiB | 43.6% done | ETA(epoch): 116s + Train loss: 0.863092 (1272.8s) ETA: 3956min + Val loss: 0.876184 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9870 t_0.4-0.6=0.8622 t_0.6-0.8=0.7285 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 131/299 (44% done) --- + [Epoch 131][10/66] loss=0.989698 avg=0.852127 VRAM=38.7GiB | 43.7% done | ETA(epoch): 1074s + [Epoch 131][20/66] loss=0.925231 avg=0.857236 VRAM=38.6GiB | 43.8% done | ETA(epoch): 890s + [Epoch 131][30/66] loss=1.016544 avg=0.859000 VRAM=38.7GiB | 43.8% done | ETA(epoch): 694s + [Epoch 131][40/66] loss=0.842433 avg=0.858440 VRAM=38.6GiB | 43.9% done | ETA(epoch): 501s + [Epoch 131][50/66] loss=0.902202 avg=0.858833 VRAM=38.7GiB | 43.9% done | ETA(epoch): 308s + [Epoch 131][60/66] loss=0.941536 avg=0.862611 VRAM=38.6GiB | 44.0% done | ETA(epoch): 116s + Train loss: 0.860191 (1270.6s) ETA: 3931min + Val loss: 0.865831 [t_0.0-0.2=1.0804 t_0.2-0.4=1.0167 t_0.4-0.6=0.7910 t_0.6-0.8=0.7032 t_0.8-1.0=0.6652] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 132/299 (44% done) --- + [Epoch 132][10/66] loss=0.800356 avg=0.845113 VRAM=38.7GiB | 44.1% done | ETA(epoch): 1087s + [Epoch 132][20/66] loss=0.859781 avg=0.848942 VRAM=38.6GiB | 44.1% done | ETA(epoch): 888s + [Epoch 132][30/66] loss=0.991624 avg=0.847441 VRAM=38.7GiB | 44.2% done | ETA(epoch): 694s + [Epoch 132][40/66] loss=0.972825 avg=0.848870 VRAM=38.6GiB | 44.2% done | ETA(epoch): 501s + [Epoch 132][50/66] loss=0.883914 avg=0.848069 VRAM=38.7GiB | 44.3% done | ETA(epoch): 308s + [Epoch 132][60/66] loss=0.891214 avg=0.843566 VRAM=38.6GiB | 44.3% done | ETA(epoch): 115s + Train loss: 0.846584 (1269.0s) ETA: 3907min + Val loss: 0.889135 [t_0.0-0.2=1.0570 t_0.2-0.4=0.9999 t_0.4-0.6=0.8855 t_0.6-0.8=0.7102 t_0.8-1.0=0.6575] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 133/299 (44% done) --- + [Epoch 133][10/66] loss=0.956566 avg=0.860314 VRAM=38.7GiB | 44.4% done | ETA(epoch): 1073s + [Epoch 133][20/66] loss=0.850479 avg=0.867388 VRAM=38.6GiB | 44.4% done | ETA(epoch): 884s + [Epoch 133][30/66] loss=0.899327 avg=0.868252 VRAM=38.7GiB | 44.5% done | ETA(epoch): 691s + [Epoch 133][40/66] loss=0.891707 avg=0.860885 VRAM=38.6GiB | 44.5% done | ETA(epoch): 499s + [Epoch 133][50/66] loss=0.836573 avg=0.861286 VRAM=38.7GiB | 44.6% done | ETA(epoch): 307s + [Epoch 133][60/66] loss=0.888073 avg=0.866372 VRAM=38.6GiB | 44.6% done | ETA(epoch): 115s + Train loss: 0.865684 (1268.8s) ETA: 3883min + Val loss: 0.862414 [t_0.0-0.2=1.0861 t_0.2-0.4=0.9537 t_0.4-0.6=0.8348 t_0.6-0.8=0.7298 t_0.8-1.0=0.6493] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 134/299 (45% done) --- + [Epoch 134][10/66] loss=0.991771 avg=0.887274 VRAM=38.7GiB | 44.7% done | ETA(epoch): 1073s + [Epoch 134][20/66] loss=0.738085 avg=0.845196 VRAM=38.6GiB | 44.8% done | ETA(epoch): 884s + [Epoch 134][30/66] loss=0.840095 avg=0.843806 VRAM=38.7GiB | 44.8% done | ETA(epoch): 692s + [Epoch 134][40/66] loss=0.859215 avg=0.852589 VRAM=38.6GiB | 44.9% done | ETA(epoch): 500s + [Epoch 134][50/66] loss=1.001384 avg=0.861500 VRAM=38.7GiB | 44.9% done | ETA(epoch): 307s + [Epoch 134][60/66] loss=0.743269 avg=0.864685 VRAM=38.6GiB | 45.0% done | ETA(epoch): 115s + Train loss: 0.866645 (1267.8s) ETA: 3859min + Val loss: 0.847219 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9758 t_0.4-0.6=0.8550 t_0.6-0.8=0.7193 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 135/299 (45% done) --- + [Epoch 135][10/66] loss=0.853054 avg=0.902341 VRAM=38.7GiB | 45.1% done | ETA(epoch): 1075s + [Epoch 135][20/66] loss=0.905486 avg=0.883891 VRAM=38.6GiB | 45.1% done | ETA(epoch): 883s + [Epoch 135][30/66] loss=0.795568 avg=0.871350 VRAM=38.7GiB | 45.2% done | ETA(epoch): 692s + [Epoch 135][40/66] loss=0.869360 avg=0.864799 VRAM=38.6GiB | 45.2% done | ETA(epoch): 499s + [Epoch 135][50/66] loss=0.736750 avg=0.865950 VRAM=38.7GiB | 45.3% done | ETA(epoch): 308s + [Epoch 135][60/66] loss=0.878589 avg=0.861073 VRAM=38.6GiB | 45.3% done | ETA(epoch): 115s + Train loss: 0.864080 (1270.0s) ETA: 3835min + Val loss: 0.838219 [t_0.0-0.2=1.0664 t_0.2-0.4=0.9653 t_0.4-0.6=0.8350 t_0.6-0.8=0.7334 t_0.8-1.0=0.6441] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 136/299 (45% done) --- + [Epoch 136][10/66] loss=0.959617 avg=0.863186 VRAM=38.7GiB | 45.4% done | ETA(epoch): 1073s + [Epoch 136][20/66] loss=0.826728 avg=0.840784 VRAM=38.6GiB | 45.4% done | ETA(epoch): 883s + [Epoch 136][30/66] loss=0.848014 avg=0.849039 VRAM=38.7GiB | 45.5% done | ETA(epoch): 691s + [Epoch 136][40/66] loss=0.881444 avg=0.861057 VRAM=38.6GiB | 45.5% done | ETA(epoch): 500s + [Epoch 136][50/66] loss=0.788363 avg=0.856458 VRAM=38.7GiB | 45.6% done | ETA(epoch): 308s + [Epoch 136][60/66] loss=0.982032 avg=0.852112 VRAM=38.6GiB | 45.6% done | ETA(epoch): 115s + Train loss: 0.850879 (1269.3s) ETA: 3811min + Val loss: 0.861876 [t_0.0-0.2=1.0691 t_0.2-0.4=0.9963 t_0.4-0.6=0.8482 t_0.6-0.8=0.7130 t_0.8-1.0=0.6488] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 137/299 (46% done) --- + [Epoch 137][10/66] loss=0.818063 avg=0.861223 VRAM=38.7GiB | 45.7% done | ETA(epoch): 1074s + [Epoch 137][20/66] loss=0.907209 avg=0.857894 VRAM=38.6GiB | 45.8% done | ETA(epoch): 883s + [Epoch 137][30/66] loss=0.862026 avg=0.865609 VRAM=38.7GiB | 45.8% done | ETA(epoch): 693s + [Epoch 137][40/66] loss=0.805498 avg=0.861224 VRAM=38.6GiB | 45.9% done | ETA(epoch): 500s + [Epoch 137][50/66] loss=0.887009 avg=0.864779 VRAM=38.7GiB | 45.9% done | ETA(epoch): 308s + [Epoch 137][60/66] loss=0.870269 avg=0.865492 VRAM=38.6GiB | 46.0% done | ETA(epoch): 115s + Train loss: 0.867499 (1268.8s) ETA: 3787min + Val loss: 0.831874 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9800 t_0.4-0.6=0.8559 t_0.6-0.8=0.7098 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 138/299 (46% done) --- + [Epoch 138][10/66] loss=0.902750 avg=0.904934 VRAM=38.7GiB | 46.1% done | ETA(epoch): 1075s + [Epoch 138][20/66] loss=0.905828 avg=0.869972 VRAM=38.6GiB | 46.1% done | ETA(epoch): 882s + [Epoch 138][30/66] loss=0.899193 avg=0.870966 VRAM=38.7GiB | 46.2% done | ETA(epoch): 693s + [Epoch 138][40/66] loss=0.839735 avg=0.873389 VRAM=38.6GiB | 46.2% done | ETA(epoch): 500s + [Epoch 138][50/66] loss=0.848969 avg=0.870884 VRAM=38.7GiB | 46.3% done | ETA(epoch): 308s + [Epoch 138][60/66] loss=0.864850 avg=0.866950 VRAM=38.6GiB | 46.3% done | ETA(epoch): 115s + Train loss: 0.870549 (1269.1s) ETA: 3762min + Val loss: 0.839346 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9866 t_0.4-0.6=0.8293 t_0.6-0.8=0.7095 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 139/299 (46% done) --- + [Epoch 139][10/66] loss=0.901214 avg=0.862231 VRAM=38.7GiB | 46.4% done | ETA(epoch): 1075s + [Epoch 139][20/66] loss=0.851425 avg=0.845783 VRAM=38.6GiB | 46.4% done | ETA(epoch): 891s + [Epoch 139][30/66] loss=0.812633 avg=0.851471 VRAM=38.7GiB | 46.5% done | ETA(epoch): 695s + [Epoch 139][40/66] loss=0.849438 avg=0.855949 VRAM=38.6GiB | 46.5% done | ETA(epoch): 502s + [Epoch 139][50/66] loss=0.855815 avg=0.852220 VRAM=38.7GiB | 46.6% done | ETA(epoch): 308s + [Epoch 139][60/66] loss=0.968666 avg=0.851877 VRAM=38.6GiB | 46.6% done | ETA(epoch): 116s + Train loss: 0.850227 (1270.5s) ETA: 3738min + Val loss: 0.873336 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9991 t_0.4-0.6=0.8500 t_0.6-0.8=0.7248 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 140/299 (47% done) --- + [Epoch 140][10/66] loss=0.901799 avg=0.871104 VRAM=38.7GiB | 46.7% done | ETA(epoch): 1085s + [Epoch 140][20/66] loss=0.948411 avg=0.864758 VRAM=38.6GiB | 46.8% done | ETA(epoch): 888s + [Epoch 140][30/66] loss=0.940087 avg=0.862760 VRAM=38.7GiB | 46.8% done | ETA(epoch): 694s + [Epoch 140][40/66] loss=0.891742 avg=0.871186 VRAM=38.6GiB | 46.9% done | ETA(epoch): 501s + [Epoch 140][50/66] loss=0.888804 avg=0.877870 VRAM=38.7GiB | 46.9% done | ETA(epoch): 308s + [Epoch 140][60/66] loss=0.837074 avg=0.872491 VRAM=38.6GiB | 47.0% done | ETA(epoch): 115s + Train loss: 0.871418 (1269.3s) ETA: 3714min + Val loss: 0.864244 [t_0.0-0.2=1.0589 t_0.2-0.4=1.0029 t_0.4-0.6=0.8275 t_0.6-0.8=0.7140 t_0.8-1.0=0.6402] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 141/299 (47% done) --- + [Epoch 141][10/66] loss=0.912558 avg=0.860790 VRAM=38.7GiB | 47.1% done | ETA(epoch): 1073s + [Epoch 141][20/66] loss=0.809460 avg=0.857536 VRAM=38.6GiB | 47.1% done | ETA(epoch): 883s + [Epoch 141][30/66] loss=0.834502 avg=0.863831 VRAM=38.7GiB | 47.2% done | ETA(epoch): 691s + [Epoch 141][40/66] loss=0.795306 avg=0.857551 VRAM=38.6GiB | 47.2% done | ETA(epoch): 499s + [Epoch 141][50/66] loss=0.796482 avg=0.858078 VRAM=38.7GiB | 47.3% done | ETA(epoch): 307s + [Epoch 141][60/66] loss=0.764161 avg=0.854799 VRAM=38.6GiB | 47.3% done | ETA(epoch): 117s + Train loss: 0.857574 (1285.3s) ETA: 3691min + Val loss: 0.855473 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9940 t_0.4-0.6=0.8742 t_0.6-0.8=0.7143 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 142/299 (47% done) --- + [Epoch 142][10/66] loss=0.875233 avg=0.892676 VRAM=38.7GiB | 47.4% done | ETA(epoch): 1074s + [Epoch 142][20/66] loss=0.961545 avg=0.882306 VRAM=38.6GiB | 47.4% done | ETA(epoch): 883s + [Epoch 142][30/66] loss=0.860704 avg=0.863264 VRAM=38.7GiB | 47.5% done | ETA(epoch): 691s + [Epoch 142][40/66] loss=0.830947 avg=0.864304 VRAM=38.6GiB | 47.5% done | ETA(epoch): 499s + [Epoch 142][50/66] loss=0.727884 avg=0.854029 VRAM=38.7GiB | 47.6% done | ETA(epoch): 307s + [Epoch 142][60/66] loss=0.895241 avg=0.855101 VRAM=38.6GiB | 47.6% done | ETA(epoch): 117s + Train loss: 0.856362 (1285.5s) ETA: 3667min + Val loss: 0.864323 [t_0.0-0.2=1.0784 t_0.2-0.4=0.9816 t_0.4-0.6=0.8297 t_0.6-0.8=0.7064 t_0.8-1.0=0.6383] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 143/299 (48% done) --- + [Epoch 143][10/66] loss=0.808544 avg=0.868695 VRAM=38.7GiB | 47.7% done | ETA(epoch): 1075s + [Epoch 143][20/66] loss=0.930405 avg=0.856382 VRAM=38.6GiB | 47.8% done | ETA(epoch): 883s + [Epoch 143][30/66] loss=0.805414 avg=0.849221 VRAM=38.7GiB | 47.8% done | ETA(epoch): 691s + [Epoch 143][40/66] loss=0.811764 avg=0.853111 VRAM=38.6GiB | 47.9% done | ETA(epoch): 499s + [Epoch 143][50/66] loss=0.904735 avg=0.860581 VRAM=38.7GiB | 47.9% done | ETA(epoch): 308s + [Epoch 143][60/66] loss=0.770185 avg=0.858048 VRAM=38.6GiB | 48.0% done | ETA(epoch): 116s + Train loss: 0.860843 (1270.4s) ETA: 3643min + Val loss: 0.865234 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9750 t_0.4-0.6=0.8366 t_0.6-0.8=0.7300 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 144/299 (48% done) --- + [Epoch 144][10/66] loss=0.821149 avg=0.855223 VRAM=38.7GiB | 48.1% done | ETA(epoch): 1074s + [Epoch 144][20/66] loss=0.845649 avg=0.867670 VRAM=38.6GiB | 48.1% done | ETA(epoch): 884s + [Epoch 144][30/66] loss=0.912701 avg=0.864565 VRAM=38.7GiB | 48.2% done | ETA(epoch): 692s + [Epoch 144][40/66] loss=0.927282 avg=0.857386 VRAM=38.6GiB | 48.2% done | ETA(epoch): 499s + [Epoch 144][50/66] loss=0.930997 avg=0.862636 VRAM=38.7GiB | 48.3% done | ETA(epoch): 307s + [Epoch 144][60/66] loss=0.857192 avg=0.858939 VRAM=38.6GiB | 48.3% done | ETA(epoch): 115s + Train loss: 0.858884 (1267.8s) ETA: 3619min + Val loss: 0.887668 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0042 t_0.4-0.6=0.8431 t_0.6-0.8=0.6993 t_0.8-1.0=0.6358] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 145/299 (48% done) --- + [Epoch 145][10/66] loss=0.737408 avg=0.850869 VRAM=38.7GiB | 48.4% done | ETA(epoch): 1073s + [Epoch 145][20/66] loss=0.937545 avg=0.864640 VRAM=38.6GiB | 48.4% done | ETA(epoch): 882s + [Epoch 145][30/66] loss=0.814080 avg=0.866551 VRAM=38.7GiB | 48.5% done | ETA(epoch): 691s + [Epoch 145][40/66] loss=0.810177 avg=0.864017 VRAM=38.6GiB | 48.5% done | ETA(epoch): 499s + [Epoch 145][50/66] loss=0.971861 avg=0.860656 VRAM=38.7GiB | 48.6% done | ETA(epoch): 307s + [Epoch 145][60/66] loss=0.952706 avg=0.862345 VRAM=38.6GiB | 48.6% done | ETA(epoch): 115s + Train loss: 0.861790 (1267.1s) ETA: 3595min + Val loss: 0.879557 [t_0.0-0.2=1.0913 t_0.2-0.4=0.9672 t_0.4-0.6=0.8505 t_0.6-0.8=0.7272 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 146/299 (49% done) --- + [Epoch 146][10/66] loss=0.847369 avg=0.868453 VRAM=38.7GiB | 48.7% done | ETA(epoch): 1075s + [Epoch 146][20/66] loss=0.893762 avg=0.863773 VRAM=38.6GiB | 48.8% done | ETA(epoch): 883s + [Epoch 146][30/66] loss=0.938526 avg=0.861471 VRAM=38.7GiB | 48.8% done | ETA(epoch): 691s + [Epoch 146][40/66] loss=0.902970 avg=0.858971 VRAM=38.6GiB | 48.9% done | ETA(epoch): 499s + [Epoch 146][50/66] loss=0.826448 avg=0.858673 VRAM=38.7GiB | 48.9% done | ETA(epoch): 307s + [Epoch 146][60/66] loss=0.758255 avg=0.860311 VRAM=38.6GiB | 49.0% done | ETA(epoch): 115s + Train loss: 0.861665 (1267.5s) ETA: 3571min + Val loss: 0.837710 [t_0.0-0.2=1.0527 t_0.2-0.4=0.9970 t_0.4-0.6=0.8778 t_0.6-0.8=0.7042 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 147/299 (49% done) --- + [Epoch 147][10/66] loss=0.746488 avg=0.858908 VRAM=38.7GiB | 49.1% done | ETA(epoch): 1089s + [Epoch 147][20/66] loss=0.891003 avg=0.860095 VRAM=38.6GiB | 49.1% done | ETA(epoch): 889s + [Epoch 147][30/66] loss=0.739864 avg=0.852898 VRAM=38.7GiB | 49.2% done | ETA(epoch): 694s + [Epoch 147][40/66] loss=0.796761 avg=0.851988 VRAM=38.6GiB | 49.2% done | ETA(epoch): 501s + [Epoch 147][50/66] loss=0.975580 avg=0.858432 VRAM=38.7GiB | 49.3% done | ETA(epoch): 308s + [Epoch 147][60/66] loss=0.843401 avg=0.862852 VRAM=38.6GiB | 49.3% done | ETA(epoch): 115s + Train loss: 0.863965 (1270.0s) ETA: 3547min + Val loss: 0.874916 [t_0.0-0.2=1.0719 t_0.2-0.4=1.0474 t_0.4-0.6=0.8285 t_0.6-0.8=0.6820 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 148/299 (49% done) --- + [Epoch 148][10/66] loss=0.884268 avg=0.853974 VRAM=38.7GiB | 49.4% done | ETA(epoch): 1074s + [Epoch 148][20/66] loss=0.843240 avg=0.866986 VRAM=38.6GiB | 49.4% done | ETA(epoch): 883s + [Epoch 148][30/66] loss=0.788094 avg=0.861633 VRAM=38.7GiB | 49.5% done | ETA(epoch): 691s + [Epoch 148][40/66] loss=0.859163 avg=0.866014 VRAM=38.6GiB | 49.5% done | ETA(epoch): 499s + [Epoch 148][50/66] loss=0.891541 avg=0.864553 VRAM=38.7GiB | 49.6% done | ETA(epoch): 307s + [Epoch 148][60/66] loss=0.855271 avg=0.866418 VRAM=38.6GiB | 49.6% done | ETA(epoch): 115s + Train loss: 0.864066 (1267.1s) ETA: 3523min + Val loss: 0.848182 [t_0.0-0.2=1.0600 t_0.2-0.4=0.9788 t_0.4-0.6=0.8348 t_0.6-0.8=0.7183 t_0.8-1.0=0.6292] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 149/299 (50% done) --- + [Epoch 149][10/66] loss=0.844031 avg=0.853861 VRAM=38.7GiB | 49.7% done | ETA(epoch): 1075s + [Epoch 149][20/66] loss=0.800901 avg=0.858420 VRAM=38.6GiB | 49.8% done | ETA(epoch): 883s + [Epoch 149][30/66] loss=0.722244 avg=0.867250 VRAM=38.7GiB | 49.8% done | ETA(epoch): 691s + [Epoch 149][40/66] loss=0.881815 avg=0.864655 VRAM=38.6GiB | 49.9% done | ETA(epoch): 499s + [Epoch 149][50/66] loss=0.861863 avg=0.859341 VRAM=38.7GiB | 49.9% done | ETA(epoch): 307s + [Epoch 149][60/66] loss=0.995830 avg=0.861017 VRAM=38.6GiB | 50.0% done | ETA(epoch): 115s + Train loss: 0.858984 (1267.5s) ETA: 3499min + Val loss: 0.894839 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9771 t_0.4-0.6=0.8641 t_0.6-0.8=0.7308 t_0.8-1.0=0.6453] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 150/299 (50% done) --- + [MilestoneVis] train_0 step 9900 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9900 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9900 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9900 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9900 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9900 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9900 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9900 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9900 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9900 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/66] loss=0.838068 avg=0.881550 VRAM=38.7GiB | 50.1% done | ETA(epoch): 6871s + [Epoch 150][20/66] loss=0.816628 avg=0.855871 VRAM=38.6GiB | 50.1% done | ETA(epoch): 3263s + [Epoch 150][30/66] loss=0.958540 avg=0.858314 VRAM=38.7GiB | 50.2% done | ETA(epoch): 1932s + [Epoch 150][40/66] loss=0.797938 avg=0.864659 VRAM=38.6GiB | 50.2% done | ETA(epoch): 1171s + [Epoch 150][50/66] loss=0.833752 avg=0.860702 VRAM=38.7GiB | 50.3% done | ETA(epoch): 638s + [Epoch 150][60/66] loss=0.940688 avg=0.857925 VRAM=38.6GiB | 50.3% done | ETA(epoch): 219s + Train loss: 0.857955 (2303.5s) ETA: 3492min + Val loss: 0.846838 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0094 t_0.4-0.6=0.8416 t_0.6-0.8=0.7149 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 151/299 (50% done) --- + [Epoch 151][10/66] loss=0.804360 avg=0.829310 VRAM=38.7GiB | 50.4% done | ETA(epoch): 1074s + [Epoch 151][20/66] loss=0.947116 avg=0.843487 VRAM=38.6GiB | 50.4% done | ETA(epoch): 883s + [Epoch 151][30/66] loss=0.769513 avg=0.834911 VRAM=38.7GiB | 50.5% done | ETA(epoch): 691s + [Epoch 151][40/66] loss=0.840649 avg=0.840960 VRAM=38.6GiB | 50.5% done | ETA(epoch): 499s + [Epoch 151][50/66] loss=0.901710 avg=0.841340 VRAM=38.7GiB | 50.6% done | ETA(epoch): 308s + [Epoch 151][60/66] loss=0.844185 avg=0.843306 VRAM=38.6GiB | 50.6% done | ETA(epoch): 115s + Train loss: 0.843377 (1268.7s) ETA: 3468min + Val loss: 0.856354 [t_0.0-0.2=1.0746 t_0.2-0.4=1.0054 t_0.4-0.6=0.8343 t_0.6-0.8=0.6963 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 152/299 (51% done) --- + [Epoch 152][10/66] loss=0.846585 avg=0.854807 VRAM=38.7GiB | 50.7% done | ETA(epoch): 1074s + [Epoch 152][20/66] loss=0.896288 avg=0.857008 VRAM=38.6GiB | 50.8% done | ETA(epoch): 883s + [Epoch 152][30/66] loss=0.816864 avg=0.855752 VRAM=38.7GiB | 50.8% done | ETA(epoch): 691s + [Epoch 152][40/66] loss=0.988895 avg=0.855990 VRAM=38.6GiB | 50.9% done | ETA(epoch): 499s + [Epoch 152][50/66] loss=0.776015 avg=0.851934 VRAM=38.7GiB | 50.9% done | ETA(epoch): 307s + [Epoch 152][60/66] loss=0.913755 avg=0.851014 VRAM=38.6GiB | 51.0% done | ETA(epoch): 115s + Train loss: 0.856853 (1266.6s) ETA: 3444min + Val loss: 0.835889 [t_0.0-0.2=1.0929 t_0.2-0.4=1.0025 t_0.4-0.6=0.8207 t_0.6-0.8=0.7152 t_0.8-1.0=0.6466] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 153/299 (51% done) --- + [Epoch 153][10/66] loss=0.926080 avg=0.888444 VRAM=38.7GiB | 51.1% done | ETA(epoch): 1078s + [Epoch 153][20/66] loss=0.893441 avg=0.896440 VRAM=38.6GiB | 51.1% done | ETA(epoch): 884s + [Epoch 153][30/66] loss=0.915100 avg=0.880670 VRAM=38.7GiB | 51.2% done | ETA(epoch): 692s + [Epoch 153][40/66] loss=0.792522 avg=0.876924 VRAM=38.6GiB | 51.2% done | ETA(epoch): 501s + [Epoch 153][50/66] loss=0.805794 avg=0.867230 VRAM=38.7GiB | 51.3% done | ETA(epoch): 308s + [Epoch 153][60/66] loss=0.911154 avg=0.870351 VRAM=38.6GiB | 51.3% done | ETA(epoch): 115s + Train loss: 0.870370 (1269.8s) ETA: 3420min + Val loss: 0.834319 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9750 t_0.4-0.6=0.8271 t_0.6-0.8=0.7100 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 154/299 (51% done) --- + [Epoch 154][10/66] loss=0.883493 avg=0.821877 VRAM=38.7GiB | 51.4% done | ETA(epoch): 1073s + [Epoch 154][20/66] loss=0.733045 avg=0.821042 VRAM=38.6GiB | 51.4% done | ETA(epoch): 882s + [Epoch 154][30/66] loss=0.883751 avg=0.824927 VRAM=38.7GiB | 51.5% done | ETA(epoch): 691s + [Epoch 154][40/66] loss=0.916192 avg=0.839959 VRAM=38.6GiB | 51.5% done | ETA(epoch): 499s + [Epoch 154][50/66] loss=0.940169 avg=0.851316 VRAM=38.7GiB | 51.6% done | ETA(epoch): 307s + [Epoch 154][60/66] loss=0.808320 avg=0.851731 VRAM=38.6GiB | 51.6% done | ETA(epoch): 115s + Train loss: 0.851584 (1267.4s) ETA: 3396min + Val loss: 0.859099 [t_0.0-0.2=1.0813 t_0.2-0.4=1.0336 t_0.4-0.6=0.8298 t_0.6-0.8=0.7247 t_0.8-1.0=0.6570] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 155/299 (52% done) --- + [Epoch 155][10/66] loss=0.762682 avg=0.862605 VRAM=38.7GiB | 51.7% done | ETA(epoch): 1076s + [Epoch 155][20/66] loss=0.910397 avg=0.860523 VRAM=38.6GiB | 51.8% done | ETA(epoch): 884s + [Epoch 155][30/66] loss=0.798733 avg=0.856137 VRAM=38.7GiB | 51.8% done | ETA(epoch): 691s + [Epoch 155][40/66] loss=0.768300 avg=0.852359 VRAM=38.6GiB | 51.9% done | ETA(epoch): 499s + [Epoch 155][50/66] loss=0.915288 avg=0.858707 VRAM=38.7GiB | 51.9% done | ETA(epoch): 307s + [Epoch 155][60/66] loss=0.871054 avg=0.860693 VRAM=38.6GiB | 52.0% done | ETA(epoch): 115s + Train loss: 0.855476 (1267.2s) ETA: 3371min + Val loss: 0.869013 [t_0.0-0.2=1.0786 t_0.2-0.4=1.0072 t_0.4-0.6=0.8656 t_0.6-0.8=0.7126 t_0.8-1.0=0.6438] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 156/299 (52% done) --- + [Epoch 156][10/66] loss=0.868720 avg=0.836014 VRAM=38.7GiB | 52.1% done | ETA(epoch): 1086s + [Epoch 156][20/66] loss=0.803103 avg=0.829413 VRAM=38.6GiB | 52.1% done | ETA(epoch): 887s + [Epoch 156][30/66] loss=0.968656 avg=0.836226 VRAM=38.7GiB | 52.2% done | ETA(epoch): 693s + [Epoch 156][40/66] loss=0.819499 avg=0.828660 VRAM=38.6GiB | 52.2% done | ETA(epoch): 500s + [Epoch 156][50/66] loss=0.759817 avg=0.839866 VRAM=38.7GiB | 52.3% done | ETA(epoch): 308s + [Epoch 156][60/66] loss=0.868894 avg=0.844221 VRAM=38.6GiB | 52.3% done | ETA(epoch): 115s + Train loss: 0.845542 (1268.3s) ETA: 3347min + Val loss: 0.854805 [t_0.0-0.2=1.0636 t_0.2-0.4=0.9765 t_0.4-0.6=0.8122 t_0.6-0.8=0.7595 t_0.8-1.0=0.6378] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 157/299 (52% done) --- + [Epoch 157][10/66] loss=0.707591 avg=0.846106 VRAM=38.7GiB | 52.4% done | ETA(epoch): 1076s + [Epoch 157][20/66] loss=0.780758 avg=0.855730 VRAM=38.6GiB | 52.4% done | ETA(epoch): 884s + [Epoch 157][30/66] loss=0.782860 avg=0.864969 VRAM=38.7GiB | 52.5% done | ETA(epoch): 691s + [Epoch 157][40/66] loss=0.837113 avg=0.860047 VRAM=38.6GiB | 52.5% done | ETA(epoch): 500s + [Epoch 157][50/66] loss=0.919963 avg=0.865121 VRAM=38.7GiB | 52.6% done | ETA(epoch): 307s + [Epoch 157][60/66] loss=0.751003 avg=0.861157 VRAM=38.6GiB | 52.6% done | ETA(epoch): 115s + Train loss: 0.857571 (1267.6s) ETA: 3323min + Val loss: 0.868392 [t_0.0-0.2=1.0922 t_0.2-0.4=0.9826 t_0.4-0.6=0.8610 t_0.6-0.8=0.7136 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 158/299 (53% done) --- + [Epoch 158][10/66] loss=0.741228 avg=0.855774 VRAM=38.7GiB | 52.7% done | ETA(epoch): 1076s + [Epoch 158][20/66] loss=0.869809 avg=0.849961 VRAM=38.6GiB | 52.8% done | ETA(epoch): 883s + [Epoch 158][30/66] loss=0.921165 avg=0.849036 VRAM=38.7GiB | 52.8% done | ETA(epoch): 691s + [Epoch 158][40/66] loss=0.915088 avg=0.856459 VRAM=38.6GiB | 52.9% done | ETA(epoch): 499s + [Epoch 158][50/66] loss=0.817827 avg=0.856183 VRAM=38.7GiB | 52.9% done | ETA(epoch): 307s + [Epoch 158][60/66] loss=0.779271 avg=0.860685 VRAM=38.6GiB | 53.0% done | ETA(epoch): 115s + Train loss: 0.856416 (1269.4s) ETA: 3299min + Val loss: 0.891796 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9802 t_0.4-0.6=0.8386 t_0.6-0.8=0.7206 t_0.8-1.0=0.6481] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 159/299 (53% done) --- + [Epoch 159][10/66] loss=0.898741 avg=0.835433 VRAM=38.7GiB | 53.1% done | ETA(epoch): 1075s + [Epoch 159][20/66] loss=0.848222 avg=0.866236 VRAM=38.6GiB | 53.1% done | ETA(epoch): 883s + [Epoch 159][30/66] loss=0.906501 avg=0.869421 VRAM=38.7GiB | 53.2% done | ETA(epoch): 691s + [Epoch 159][40/66] loss=0.852119 avg=0.862056 VRAM=38.6GiB | 53.2% done | ETA(epoch): 499s + [Epoch 159][50/66] loss=0.883728 avg=0.854560 VRAM=38.7GiB | 53.3% done | ETA(epoch): 307s + [Epoch 159][60/66] loss=0.938044 avg=0.851008 VRAM=38.6GiB | 53.3% done | ETA(epoch): 115s + Train loss: 0.853569 (1267.5s) ETA: 3275min + Val loss: 0.829764 [t_0.0-0.2=1.0777 t_0.2-0.4=0.9785 t_0.4-0.6=0.8423 t_0.6-0.8=0.7219 t_0.8-1.0=0.6542] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 160/299 (53% done) --- + [Epoch 160][10/66] loss=0.880304 avg=0.843756 VRAM=38.7GiB | 53.4% done | ETA(epoch): 1076s + [Epoch 160][20/66] loss=0.903366 avg=0.845336 VRAM=38.6GiB | 53.4% done | ETA(epoch): 883s + [Epoch 160][30/66] loss=0.738894 avg=0.856857 VRAM=38.7GiB | 53.5% done | ETA(epoch): 691s + [Epoch 160][40/66] loss=0.900806 avg=0.860332 VRAM=38.6GiB | 53.5% done | ETA(epoch): 500s + [Epoch 160][50/66] loss=0.750327 avg=0.858856 VRAM=38.7GiB | 53.6% done | ETA(epoch): 308s + [Epoch 160][60/66] loss=0.925989 avg=0.861054 VRAM=38.6GiB | 53.6% done | ETA(epoch): 115s + Train loss: 0.865141 (1268.9s) ETA: 3251min + Val loss: 0.854393 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9811 t_0.4-0.6=0.8676 t_0.6-0.8=0.7176 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 161/299 (54% done) --- + [Epoch 161][10/66] loss=0.741034 avg=0.848846 VRAM=38.7GiB | 53.7% done | ETA(epoch): 1075s + [Epoch 161][20/66] loss=0.838918 avg=0.853294 VRAM=38.6GiB | 53.8% done | ETA(epoch): 884s + [Epoch 161][30/66] loss=0.929287 avg=0.859388 VRAM=38.7GiB | 53.8% done | ETA(epoch): 692s + [Epoch 161][40/66] loss=0.909394 avg=0.867030 VRAM=38.6GiB | 53.9% done | ETA(epoch): 499s + [Epoch 161][50/66] loss=0.825425 avg=0.868346 VRAM=38.7GiB | 53.9% done | ETA(epoch): 307s + [Epoch 161][60/66] loss=0.909914 avg=0.862228 VRAM=38.6GiB | 54.0% done | ETA(epoch): 115s + Train loss: 0.858616 (1267.1s) ETA: 3227min + Val loss: 0.884881 [t_0.0-0.2=1.0644 t_0.2-0.4=0.9892 t_0.4-0.6=0.8456 t_0.6-0.8=0.7003 t_0.8-1.0=0.6648] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 162/299 (54% done) --- + [Epoch 162][10/66] loss=0.816809 avg=0.847957 VRAM=38.7GiB | 54.1% done | ETA(epoch): 1074s + [Epoch 162][20/66] loss=0.915873 avg=0.829990 VRAM=38.6GiB | 54.1% done | ETA(epoch): 890s + [Epoch 162][30/66] loss=0.912882 avg=0.840058 VRAM=38.7GiB | 54.2% done | ETA(epoch): 694s + [Epoch 162][40/66] loss=0.775688 avg=0.853261 VRAM=38.6GiB | 54.2% done | ETA(epoch): 501s + [Epoch 162][50/66] loss=0.892484 avg=0.857577 VRAM=38.7GiB | 54.3% done | ETA(epoch): 308s + [Epoch 162][60/66] loss=0.882275 avg=0.857024 VRAM=38.6GiB | 54.3% done | ETA(epoch): 115s + Train loss: 0.854503 (1270.2s) ETA: 3203min + Val loss: 0.869211 [t_0.0-0.2=1.0613 t_0.2-0.4=1.0214 t_0.4-0.6=0.8518 t_0.6-0.8=0.7155 t_0.8-1.0=0.6481] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 163/299 (54% done) --- + [Epoch 163][10/66] loss=0.883498 avg=0.852208 VRAM=38.7GiB | 54.4% done | ETA(epoch): 1075s + [Epoch 163][20/66] loss=0.836482 avg=0.862888 VRAM=38.6GiB | 54.4% done | ETA(epoch): 893s + [Epoch 163][30/66] loss=0.887033 avg=0.867682 VRAM=38.7GiB | 54.5% done | ETA(epoch): 696s + [Epoch 163][40/66] loss=0.828259 avg=0.870144 VRAM=38.6GiB | 54.5% done | ETA(epoch): 502s + [Epoch 163][50/66] loss=0.754036 avg=0.870305 VRAM=38.7GiB | 54.6% done | ETA(epoch): 309s + [Epoch 163][60/66] loss=0.751712 avg=0.870111 VRAM=38.6GiB | 54.6% done | ETA(epoch): 116s + Train loss: 0.866999 (1271.1s) ETA: 3180min + Val loss: 0.875448 [t_0.0-0.2=1.0596 t_0.2-0.4=1.0385 t_0.4-0.6=0.8578 t_0.6-0.8=0.7056 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 164/299 (55% done) --- + [Epoch 164][10/66] loss=0.900666 avg=0.841915 VRAM=38.7GiB | 54.7% done | ETA(epoch): 1074s + [Epoch 164][20/66] loss=0.766222 avg=0.847217 VRAM=38.6GiB | 54.8% done | ETA(epoch): 882s + [Epoch 164][30/66] loss=0.904670 avg=0.850974 VRAM=38.7GiB | 54.8% done | ETA(epoch): 691s + [Epoch 164][40/66] loss=0.849232 avg=0.852973 VRAM=38.6GiB | 54.9% done | ETA(epoch): 499s + [Epoch 164][50/66] loss=0.952110 avg=0.850365 VRAM=38.7GiB | 54.9% done | ETA(epoch): 307s + [Epoch 164][60/66] loss=0.930817 avg=0.844714 VRAM=38.6GiB | 55.0% done | ETA(epoch): 115s + Train loss: 0.849360 (1266.3s) ETA: 3156min + Val loss: 0.901186 [t_0.0-0.2=1.0864 t_0.2-0.4=0.9950 t_0.4-0.6=0.8466 t_0.6-0.8=0.7327 t_0.8-1.0=0.6397] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 165/299 (55% done) --- + [Epoch 165][10/66] loss=0.938314 avg=0.844095 VRAM=38.7GiB | 55.1% done | ETA(epoch): 1075s + [Epoch 165][20/66] loss=0.792689 avg=0.840996 VRAM=38.6GiB | 55.1% done | ETA(epoch): 883s + [Epoch 165][30/66] loss=0.894735 avg=0.855724 VRAM=38.7GiB | 55.2% done | ETA(epoch): 691s + [Epoch 165][40/66] loss=0.896473 avg=0.857527 VRAM=38.6GiB | 55.2% done | ETA(epoch): 499s + [Epoch 165][50/66] loss=0.863485 avg=0.854482 VRAM=38.7GiB | 55.3% done | ETA(epoch): 307s + [Epoch 165][60/66] loss=0.725696 avg=0.852863 VRAM=38.6GiB | 55.3% done | ETA(epoch): 115s + Train loss: 0.855900 (1267.7s) ETA: 3132min + Val loss: 0.844257 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9891 t_0.4-0.6=0.8633 t_0.6-0.8=0.7097 t_0.8-1.0=0.6318] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 166/299 (55% done) --- + [Epoch 166][10/66] loss=0.781184 avg=0.854040 VRAM=38.7GiB | 55.4% done | ETA(epoch): 1074s + [Epoch 166][20/66] loss=0.814251 avg=0.855667 VRAM=38.6GiB | 55.4% done | ETA(epoch): 883s + [Epoch 166][30/66] loss=0.887125 avg=0.869100 VRAM=38.7GiB | 55.5% done | ETA(epoch): 691s + [Epoch 166][40/66] loss=0.927168 avg=0.860162 VRAM=38.6GiB | 55.5% done | ETA(epoch): 499s + [Epoch 166][50/66] loss=0.818964 avg=0.856962 VRAM=38.7GiB | 55.6% done | ETA(epoch): 307s + [Epoch 166][60/66] loss=0.820859 avg=0.851519 VRAM=38.6GiB | 55.6% done | ETA(epoch): 115s + Train loss: 0.850119 (1267.9s) ETA: 3108min + Val loss: 0.909589 [t_0.0-0.2=1.0768 t_0.2-0.4=1.0003 t_0.4-0.6=0.8606 t_0.6-0.8=0.7148 t_0.8-1.0=0.6526] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 167/299 (56% done) --- + [Epoch 167][10/66] loss=0.847853 avg=0.852278 VRAM=38.7GiB | 55.7% done | ETA(epoch): 1075s + [Epoch 167][20/66] loss=0.784940 avg=0.850638 VRAM=38.6GiB | 55.8% done | ETA(epoch): 884s + [Epoch 167][30/66] loss=0.936194 avg=0.850415 VRAM=38.7GiB | 55.8% done | ETA(epoch): 692s + [Epoch 167][40/66] loss=0.769695 avg=0.836589 VRAM=38.6GiB | 55.9% done | ETA(epoch): 499s + [Epoch 167][50/66] loss=0.871939 avg=0.838330 VRAM=38.7GiB | 55.9% done | ETA(epoch): 307s + [Epoch 167][60/66] loss=0.902232 avg=0.843628 VRAM=38.6GiB | 56.0% done | ETA(epoch): 115s + Train loss: 0.840718 (1268.3s) ETA: 3084min + Val loss: 0.871045 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0089 t_0.4-0.6=0.8377 t_0.6-0.8=0.6967 t_0.8-1.0=0.6389] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 168/299 (56% done) --- + [Epoch 168][10/66] loss=0.845669 avg=0.841462 VRAM=38.7GiB | 56.1% done | ETA(epoch): 1074s + [Epoch 168][20/66] loss=0.847199 avg=0.851949 VRAM=38.6GiB | 56.1% done | ETA(epoch): 883s + [Epoch 168][30/66] loss=0.721718 avg=0.842653 VRAM=38.7GiB | 56.2% done | ETA(epoch): 692s + [Epoch 168][40/66] loss=0.811220 avg=0.848490 VRAM=38.6GiB | 56.2% done | ETA(epoch): 502s + [Epoch 168][50/66] loss=0.731240 avg=0.847832 VRAM=38.7GiB | 56.3% done | ETA(epoch): 308s + [Epoch 168][60/66] loss=0.795757 avg=0.847001 VRAM=38.6GiB | 56.3% done | ETA(epoch): 116s + Train loss: 0.847806 (1271.1s) ETA: 3060min + Val loss: 0.824798 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9865 t_0.4-0.6=0.8547 t_0.6-0.8=0.6899 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 169/299 (56% done) --- + [Epoch 169][10/66] loss=0.895131 avg=0.863259 VRAM=38.7GiB | 56.4% done | ETA(epoch): 1076s + [Epoch 169][20/66] loss=0.813514 avg=0.855207 VRAM=38.6GiB | 56.4% done | ETA(epoch): 884s + [Epoch 169][30/66] loss=0.849147 avg=0.863969 VRAM=38.7GiB | 56.5% done | ETA(epoch): 692s + [Epoch 169][40/66] loss=0.853686 avg=0.865387 VRAM=38.6GiB | 56.5% done | ETA(epoch): 500s + [Epoch 169][50/66] loss=0.914664 avg=0.867100 VRAM=38.7GiB | 56.6% done | ETA(epoch): 307s + [Epoch 169][60/66] loss=0.909554 avg=0.868990 VRAM=38.6GiB | 56.6% done | ETA(epoch): 115s + Train loss: 0.862983 (1267.9s) ETA: 3036min + Val loss: 0.831373 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9448 t_0.4-0.6=0.8395 t_0.6-0.8=0.7210 t_0.8-1.0=0.6357] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 170/299 (57% done) --- + [Epoch 170][10/66] loss=0.868485 avg=0.867497 VRAM=38.7GiB | 56.7% done | ETA(epoch): 1075s + [Epoch 170][20/66] loss=0.965297 avg=0.856231 VRAM=38.6GiB | 56.8% done | ETA(epoch): 883s + [Epoch 170][30/66] loss=0.914131 avg=0.857622 VRAM=38.7GiB | 56.8% done | ETA(epoch): 691s + [Epoch 170][40/66] loss=0.913562 avg=0.866552 VRAM=38.6GiB | 56.9% done | ETA(epoch): 499s + [Epoch 170][50/66] loss=0.874925 avg=0.865414 VRAM=38.7GiB | 56.9% done | ETA(epoch): 307s + [Epoch 170][60/66] loss=0.815256 avg=0.863120 VRAM=38.6GiB | 57.0% done | ETA(epoch): 115s + Train loss: 0.861924 (1266.8s) ETA: 3012min + Val loss: 0.870538 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9979 t_0.4-0.6=0.8017 t_0.6-0.8=0.7367 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 171/299 (57% done) --- + [Epoch 171][10/66] loss=0.932432 avg=0.846283 VRAM=38.7GiB | 57.1% done | ETA(epoch): 1086s + [Epoch 171][20/66] loss=0.922739 avg=0.844425 VRAM=38.6GiB | 57.1% done | ETA(epoch): 888s + [Epoch 171][30/66] loss=0.946445 avg=0.853710 VRAM=38.7GiB | 57.2% done | ETA(epoch): 693s + [Epoch 171][40/66] loss=0.781640 avg=0.850943 VRAM=38.6GiB | 57.2% done | ETA(epoch): 501s + [Epoch 171][50/66] loss=0.972077 avg=0.858298 VRAM=38.7GiB | 57.3% done | ETA(epoch): 308s + [Epoch 171][60/66] loss=0.911692 avg=0.857059 VRAM=38.6GiB | 57.3% done | ETA(epoch): 115s + Train loss: 0.859710 (1269.3s) ETA: 2989min + Val loss: 0.871369 [t_0.0-0.2=1.0694 t_0.2-0.4=1.0043 t_0.4-0.6=0.8747 t_0.6-0.8=0.7134 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 20.8/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 172/299 (57% done) --- + [Epoch 172][10/66] loss=0.762637 avg=0.888015 VRAM=38.7GiB | 57.4% done | ETA(epoch): 1073s + [Epoch 172][20/66] loss=0.796476 avg=0.876902 VRAM=38.6GiB | 57.4% done | ETA(epoch): 882s + [Epoch 172][30/66] loss=0.794500 avg=0.865711 VRAM=38.7GiB | 57.5% done | ETA(epoch): 691s + [Epoch 172][40/66] loss=0.758065 avg=0.865731 VRAM=38.6GiB | 57.5% done | ETA(epoch): 499s + [Epoch 172][50/66] loss=0.923756 avg=0.861334 VRAM=38.7GiB | 57.6% done | ETA(epoch): 307s + [Epoch 172][60/66] loss=0.908272 avg=0.860762 VRAM=38.6GiB | 57.6% done | ETA(epoch): 115s + Train loss: 0.862474 (1267.7s) ETA: 2965min + Val loss: 0.896693 [t_0.0-0.2=1.0605 t_0.2-0.4=1.0064 t_0.4-0.6=0.8845 t_0.6-0.8=0.7320 t_0.8-1.0=0.6497] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 173/299 (58% done) --- + [Epoch 173][10/66] loss=0.905197 avg=0.849841 VRAM=38.7GiB | 57.7% done | ETA(epoch): 1074s + [Epoch 173][20/66] loss=0.939411 avg=0.851971 VRAM=38.6GiB | 57.8% done | ETA(epoch): 884s + [Epoch 173][30/66] loss=0.789096 avg=0.853505 VRAM=38.7GiB | 57.8% done | ETA(epoch): 691s + [Epoch 173][40/66] loss=0.878234 avg=0.856683 VRAM=38.6GiB | 57.9% done | ETA(epoch): 499s + [Epoch 173][50/66] loss=0.973917 avg=0.861137 VRAM=38.7GiB | 57.9% done | ETA(epoch): 307s + [Epoch 173][60/66] loss=0.840749 avg=0.862553 VRAM=38.6GiB | 58.0% done | ETA(epoch): 115s + Train loss: 0.860459 (1269.4s) ETA: 2941min + Val loss: 0.871129 [t_0.0-0.2=1.0665 t_0.2-0.4=1.0061 t_0.4-0.6=0.8687 t_0.6-0.8=0.7187 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 174/299 (58% done) --- + [Epoch 174][10/66] loss=0.811518 avg=0.841816 VRAM=38.7GiB | 58.1% done | ETA(epoch): 1076s + [Epoch 174][20/66] loss=0.865572 avg=0.855995 VRAM=38.6GiB | 58.1% done | ETA(epoch): 884s + [Epoch 174][30/66] loss=0.924726 avg=0.871025 VRAM=38.7GiB | 58.2% done | ETA(epoch): 691s + [Epoch 174][40/66] loss=0.969602 avg=0.882500 VRAM=38.6GiB | 58.2% done | ETA(epoch): 499s + [Epoch 174][50/66] loss=0.872055 avg=0.875665 VRAM=38.7GiB | 58.3% done | ETA(epoch): 309s + [Epoch 174][60/66] loss=0.724013 avg=0.867688 VRAM=38.6GiB | 58.3% done | ETA(epoch): 116s + Train loss: 0.861094 (1271.7s) ETA: 2917min + Val loss: 0.848541 [t_0.0-0.2=1.0842 t_0.2-0.4=1.0000 t_0.4-0.6=0.8633 t_0.6-0.8=0.6976 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 175/299 (58% done) --- + [Epoch 175][10/66] loss=0.876824 avg=0.893582 VRAM=38.7GiB | 58.4% done | ETA(epoch): 1075s + [Epoch 175][20/66] loss=0.970693 avg=0.884827 VRAM=38.6GiB | 58.4% done | ETA(epoch): 883s + [Epoch 175][30/66] loss=0.862311 avg=0.882548 VRAM=38.7GiB | 58.5% done | ETA(epoch): 691s + [Epoch 175][40/66] loss=0.797931 avg=0.874688 VRAM=38.6GiB | 58.5% done | ETA(epoch): 502s + [Epoch 175][50/66] loss=0.886536 avg=0.877562 VRAM=38.7GiB | 58.6% done | ETA(epoch): 309s + [Epoch 175][60/66] loss=0.916695 avg=0.874536 VRAM=38.6GiB | 58.6% done | ETA(epoch): 116s + Train loss: 0.875792 (1271.2s) ETA: 2893min + Val loss: 0.840560 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9910 t_0.4-0.6=0.8590 t_0.6-0.8=0.7086 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 176/299 (59% done) --- + [Epoch 176][10/66] loss=0.903984 avg=0.838049 VRAM=38.7GiB | 58.7% done | ETA(epoch): 1074s + [Epoch 176][20/66] loss=0.885558 avg=0.873640 VRAM=38.6GiB | 58.8% done | ETA(epoch): 882s + [Epoch 176][30/66] loss=0.761692 avg=0.864262 VRAM=38.7GiB | 58.8% done | ETA(epoch): 691s + [Epoch 176][40/66] loss=0.794016 avg=0.848629 VRAM=38.6GiB | 58.9% done | ETA(epoch): 501s + [Epoch 176][50/66] loss=0.839923 avg=0.851985 VRAM=38.7GiB | 58.9% done | ETA(epoch): 308s + [Epoch 176][60/66] loss=0.862009 avg=0.856029 VRAM=38.6GiB | 59.0% done | ETA(epoch): 115s + Train loss: 0.854580 (1269.1s) ETA: 2870min + Val loss: 0.860557 [t_0.0-0.2=1.0846 t_0.2-0.4=1.0002 t_0.4-0.6=0.8636 t_0.6-0.8=0.7165 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 177/299 (59% done) --- + [Epoch 177][10/66] loss=0.853990 avg=0.829224 VRAM=38.7GiB | 59.1% done | ETA(epoch): 1075s + [Epoch 177][20/66] loss=0.818801 avg=0.813782 VRAM=38.6GiB | 59.1% done | ETA(epoch): 883s + [Epoch 177][30/66] loss=0.575452 avg=0.830054 VRAM=38.7GiB | 59.2% done | ETA(epoch): 700s + [Epoch 177][40/66] loss=0.769902 avg=0.843667 VRAM=38.6GiB | 59.2% done | ETA(epoch): 504s + [Epoch 177][50/66] loss=0.788871 avg=0.843223 VRAM=38.7GiB | 59.3% done | ETA(epoch): 310s + [Epoch 177][60/66] loss=0.946991 avg=0.842095 VRAM=38.6GiB | 59.3% done | ETA(epoch): 116s + Train loss: 0.843985 (1274.7s) ETA: 2846min + Val loss: 0.879939 [t_0.0-0.2=1.0689 t_0.2-0.4=1.0040 t_0.4-0.6=0.8396 t_0.6-0.8=0.7141 t_0.8-1.0=0.6555] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 178/299 (59% done) --- + [Epoch 178][10/66] loss=0.897687 avg=0.834032 VRAM=38.7GiB | 59.4% done | ETA(epoch): 1077s + [Epoch 178][20/66] loss=0.817502 avg=0.816259 VRAM=38.6GiB | 59.4% done | ETA(epoch): 883s + [Epoch 178][30/66] loss=0.898349 avg=0.829833 VRAM=38.7GiB | 59.5% done | ETA(epoch): 691s + [Epoch 178][40/66] loss=0.748158 avg=0.832332 VRAM=38.6GiB | 59.5% done | ETA(epoch): 499s + [Epoch 178][50/66] loss=0.842442 avg=0.837976 VRAM=38.7GiB | 59.6% done | ETA(epoch): 307s + [Epoch 178][60/66] loss=0.804461 avg=0.837326 VRAM=38.6GiB | 59.6% done | ETA(epoch): 115s + Train loss: 0.841885 (1267.1s) ETA: 2822min + Val loss: 0.900303 [t_0.0-0.2=1.0641 t_0.2-0.4=0.9852 t_0.4-0.6=0.8558 t_0.6-0.8=0.7203 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 179/299 (60% done) --- + [Epoch 179][10/66] loss=0.804972 avg=0.861195 VRAM=38.7GiB | 59.7% done | ETA(epoch): 1075s + [Epoch 179][20/66] loss=0.783153 avg=0.849156 VRAM=38.6GiB | 59.8% done | ETA(epoch): 883s + [Epoch 179][30/66] loss=0.794986 avg=0.854249 VRAM=38.7GiB | 59.8% done | ETA(epoch): 691s + [Epoch 179][40/66] loss=0.826932 avg=0.850726 VRAM=38.6GiB | 59.9% done | ETA(epoch): 499s + [Epoch 179][50/66] loss=0.953967 avg=0.852026 VRAM=38.7GiB | 59.9% done | ETA(epoch): 307s + [Epoch 179][60/66] loss=0.829885 avg=0.856682 VRAM=38.6GiB | 60.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 11879 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 11879 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 11879 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 11879 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 11879 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 11879 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 11879 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 11879 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 11879 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 11879 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.855631 (2304.6s) ETA: 2810min + Val loss: 0.842354 [t_0.0-0.2=1.0773 t_0.2-0.4=1.0111 t_0.4-0.6=0.8336 t_0.6-0.8=0.7273 t_0.8-1.0=0.6566] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 180/299 (60% done) --- + [Epoch 180][10/66] loss=0.738031 avg=0.841270 VRAM=38.7GiB | 60.1% done | ETA(epoch): 1103s + [Epoch 180][20/66] loss=0.767196 avg=0.861787 VRAM=38.6GiB | 60.1% done | ETA(epoch): 894s + [Epoch 180][30/66] loss=0.812627 avg=0.850091 VRAM=38.7GiB | 60.2% done | ETA(epoch): 697s + [Epoch 180][40/66] loss=0.835573 avg=0.843584 VRAM=38.6GiB | 60.2% done | ETA(epoch): 502s + [Epoch 180][50/66] loss=0.941306 avg=0.848637 VRAM=38.7GiB | 60.3% done | ETA(epoch): 309s + [Epoch 180][60/66] loss=0.899831 avg=0.849474 VRAM=38.6GiB | 60.3% done | ETA(epoch): 116s + Train loss: 0.847414 (1272.4s) ETA: 2786min + Val loss: 0.856684 [t_0.0-0.2=1.0597 t_0.2-0.4=1.0164 t_0.4-0.6=0.8296 t_0.6-0.8=0.7167 t_0.8-1.0=0.6551] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 181/299 (60% done) --- + [Epoch 181][10/66] loss=0.743702 avg=0.844273 VRAM=38.7GiB | 60.4% done | ETA(epoch): 1075s + [Epoch 181][20/66] loss=0.802168 avg=0.836239 VRAM=38.6GiB | 60.4% done | ETA(epoch): 883s + [Epoch 181][30/66] loss=0.746059 avg=0.860491 VRAM=38.7GiB | 60.5% done | ETA(epoch): 692s + [Epoch 181][40/66] loss=0.771457 avg=0.855588 VRAM=38.6GiB | 60.5% done | ETA(epoch): 499s + [Epoch 181][50/66] loss=0.915526 avg=0.852422 VRAM=38.7GiB | 60.6% done | ETA(epoch): 307s + [Epoch 181][60/66] loss=0.831208 avg=0.847065 VRAM=38.6GiB | 60.6% done | ETA(epoch): 115s + Train loss: 0.845400 (1268.2s) ETA: 2762min + Val loss: 0.893162 [t_0.0-0.2=1.0772 t_0.2-0.4=0.9842 t_0.4-0.6=0.8434 t_0.6-0.8=0.7127 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 182/299 (61% done) --- + [Epoch 182][10/66] loss=0.787934 avg=0.849522 VRAM=38.7GiB | 60.7% done | ETA(epoch): 1075s + [Epoch 182][20/66] loss=0.915580 avg=0.857809 VRAM=38.6GiB | 60.8% done | ETA(epoch): 883s + [Epoch 182][30/66] loss=0.909457 avg=0.861915 VRAM=38.7GiB | 60.8% done | ETA(epoch): 691s + [Epoch 182][40/66] loss=0.880637 avg=0.862971 VRAM=38.6GiB | 60.9% done | ETA(epoch): 499s + [Epoch 182][50/66] loss=0.770424 avg=0.861000 VRAM=38.7GiB | 60.9% done | ETA(epoch): 307s + [Epoch 182][60/66] loss=0.785337 avg=0.856918 VRAM=38.6GiB | 61.0% done | ETA(epoch): 115s + Train loss: 0.860493 (1267.6s) ETA: 2738min + Val loss: 0.849583 [t_0.0-0.2=1.0597 t_0.2-0.4=0.9680 t_0.4-0.6=0.8489 t_0.6-0.8=0.7273 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 183/299 (61% done) --- + [Epoch 183][10/66] loss=0.944717 avg=0.852393 VRAM=38.7GiB | 61.1% done | ETA(epoch): 1074s + [Epoch 183][20/66] loss=0.944128 avg=0.854623 VRAM=38.6GiB | 61.1% done | ETA(epoch): 882s + [Epoch 183][30/66] loss=0.858541 avg=0.852903 VRAM=38.7GiB | 61.2% done | ETA(epoch): 691s + [Epoch 183][40/66] loss=0.986144 avg=0.855213 VRAM=38.6GiB | 61.2% done | ETA(epoch): 499s + [Epoch 183][50/66] loss=0.792809 avg=0.855579 VRAM=38.7GiB | 61.3% done | ETA(epoch): 307s + [Epoch 183][60/66] loss=0.939675 avg=0.849233 VRAM=38.6GiB | 61.3% done | ETA(epoch): 115s + Train loss: 0.851519 (1266.3s) ETA: 2715min + Val loss: 0.823383 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9885 t_0.4-0.6=0.8459 t_0.6-0.8=0.6961 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 184/299 (61% done) --- + [Epoch 184][10/66] loss=0.858853 avg=0.869888 VRAM=38.7GiB | 61.4% done | ETA(epoch): 1077s + [Epoch 184][20/66] loss=0.921332 avg=0.879878 VRAM=38.6GiB | 61.4% done | ETA(epoch): 884s + [Epoch 184][30/66] loss=0.882668 avg=0.876018 VRAM=38.7GiB | 61.5% done | ETA(epoch): 691s + [Epoch 184][40/66] loss=0.834967 avg=0.866757 VRAM=38.6GiB | 61.5% done | ETA(epoch): 500s + [Epoch 184][50/66] loss=0.907827 avg=0.868695 VRAM=38.7GiB | 61.6% done | ETA(epoch): 307s + [Epoch 184][60/66] loss=0.815177 avg=0.866909 VRAM=38.6GiB | 61.6% done | ETA(epoch): 115s + Train loss: 0.866931 (1267.7s) ETA: 2691min + Val loss: 0.845956 [t_0.0-0.2=1.0777 t_0.2-0.4=0.9786 t_0.4-0.6=0.8593 t_0.6-0.8=0.6930 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 185/299 (62% done) --- + [Epoch 185][10/66] loss=0.982956 avg=0.861916 VRAM=38.7GiB | 61.7% done | ETA(epoch): 1073s + [Epoch 185][20/66] loss=0.754875 avg=0.848838 VRAM=38.6GiB | 61.8% done | ETA(epoch): 882s + [Epoch 185][30/66] loss=0.928185 avg=0.854290 VRAM=38.7GiB | 61.8% done | ETA(epoch): 695s + [Epoch 185][40/66] loss=0.797332 avg=0.851200 VRAM=38.6GiB | 61.9% done | ETA(epoch): 501s + [Epoch 185][50/66] loss=0.971149 avg=0.854027 VRAM=38.7GiB | 61.9% done | ETA(epoch): 308s + [Epoch 185][60/66] loss=0.815317 avg=0.854108 VRAM=38.6GiB | 62.0% done | ETA(epoch): 116s + Train loss: 0.863098 (1270.6s) ETA: 2667min + Val loss: 0.827614 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0045 t_0.4-0.6=0.8392 t_0.6-0.8=0.7122 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 186/299 (62% done) --- + [Epoch 186][10/66] loss=0.821700 avg=0.849597 VRAM=38.7GiB | 62.1% done | ETA(epoch): 1074s + [Epoch 186][20/66] loss=0.837815 avg=0.839136 VRAM=38.6GiB | 62.1% done | ETA(epoch): 883s + [Epoch 186][30/66] loss=0.775850 avg=0.831404 VRAM=38.7GiB | 62.2% done | ETA(epoch): 691s + [Epoch 186][40/66] loss=0.898412 avg=0.839812 VRAM=38.6GiB | 62.2% done | ETA(epoch): 499s + [Epoch 186][50/66] loss=0.745044 avg=0.840062 VRAM=38.7GiB | 62.3% done | ETA(epoch): 307s + [Epoch 186][60/66] loss=0.807987 avg=0.840166 VRAM=38.6GiB | 62.3% done | ETA(epoch): 115s + Train loss: 0.842886 (1267.1s) ETA: 2643min + Val loss: 0.823364 [t_0.0-0.2=1.0680 t_0.2-0.4=1.0259 t_0.4-0.6=0.8316 t_0.6-0.8=0.7190 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 187/299 (62% done) --- + [Epoch 187][10/66] loss=0.922393 avg=0.835127 VRAM=38.7GiB | 62.4% done | ETA(epoch): 1075s + [Epoch 187][20/66] loss=0.741876 avg=0.833195 VRAM=38.6GiB | 62.4% done | ETA(epoch): 882s + [Epoch 187][30/66] loss=0.936162 avg=0.839975 VRAM=38.7GiB | 62.5% done | ETA(epoch): 690s + [Epoch 187][40/66] loss=0.796265 avg=0.855509 VRAM=38.6GiB | 62.5% done | ETA(epoch): 499s + [Epoch 187][50/66] loss=1.019448 avg=0.863511 VRAM=38.7GiB | 62.6% done | ETA(epoch): 307s + [Epoch 187][60/66] loss=0.947645 avg=0.863459 VRAM=38.6GiB | 62.6% done | ETA(epoch): 115s + Train loss: 0.864954 (1266.7s) ETA: 2619min + Val loss: 0.882447 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0188 t_0.4-0.6=0.8623 t_0.6-0.8=0.7224 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 188/299 (63% done) --- + [Epoch 188][10/66] loss=0.727071 avg=0.895041 VRAM=38.7GiB | 62.7% done | ETA(epoch): 1075s + [Epoch 188][20/66] loss=0.913729 avg=0.902711 VRAM=38.6GiB | 62.8% done | ETA(epoch): 882s + [Epoch 188][30/66] loss=0.899100 avg=0.888175 VRAM=38.7GiB | 62.8% done | ETA(epoch): 691s + [Epoch 188][40/66] loss=0.779958 avg=0.884540 VRAM=38.6GiB | 62.9% done | ETA(epoch): 499s + [Epoch 188][50/66] loss=0.817509 avg=0.878868 VRAM=38.7GiB | 62.9% done | ETA(epoch): 307s + [Epoch 188][60/66] loss=0.936804 avg=0.876402 VRAM=38.6GiB | 63.0% done | ETA(epoch): 115s + Train loss: 0.872193 (1269.4s) ETA: 2596min + Val loss: 0.832153 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0026 t_0.4-0.6=0.8495 t_0.6-0.8=0.6918 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 189/299 (63% done) --- + [Epoch 189][10/66] loss=0.946883 avg=0.883663 VRAM=38.7GiB | 63.1% done | ETA(epoch): 1075s + [Epoch 189][20/66] loss=0.870860 avg=0.855942 VRAM=38.6GiB | 63.1% done | ETA(epoch): 883s + [Epoch 189][30/66] loss=0.901439 avg=0.856316 VRAM=38.7GiB | 63.2% done | ETA(epoch): 691s + [Epoch 189][40/66] loss=0.832783 avg=0.856430 VRAM=38.6GiB | 63.2% done | ETA(epoch): 499s + [Epoch 189][50/66] loss=0.737660 avg=0.852142 VRAM=38.7GiB | 63.3% done | ETA(epoch): 307s + [Epoch 189][60/66] loss=0.888408 avg=0.853829 VRAM=38.6GiB | 63.3% done | ETA(epoch): 115s + Train loss: 0.857908 (1268.2s) ETA: 2572min + Val loss: 0.866468 [t_0.0-0.2=1.0838 t_0.2-0.4=0.9547 t_0.4-0.6=0.8835 t_0.6-0.8=0.6838 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 190/299 (63% done) --- + [Epoch 190][10/66] loss=0.846465 avg=0.861610 VRAM=38.7GiB | 63.4% done | ETA(epoch): 1074s + [Epoch 190][20/66] loss=0.853072 avg=0.863745 VRAM=38.6GiB | 63.4% done | ETA(epoch): 883s + [Epoch 190][30/66] loss=0.860583 avg=0.855660 VRAM=38.7GiB | 63.5% done | ETA(epoch): 691s + [Epoch 190][40/66] loss=0.881006 avg=0.866322 VRAM=38.6GiB | 63.5% done | ETA(epoch): 499s + [Epoch 190][50/66] loss=0.954042 avg=0.856566 VRAM=38.7GiB | 63.6% done | ETA(epoch): 308s + [Epoch 190][60/66] loss=0.812627 avg=0.856437 VRAM=38.6GiB | 63.6% done | ETA(epoch): 115s + Train loss: 0.857803 (1268.7s) ETA: 2548min + Val loss: 0.873669 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0107 t_0.4-0.6=0.8325 t_0.6-0.8=0.7092 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 191/299 (64% done) --- + [Epoch 191][10/66] loss=0.784366 avg=0.861436 VRAM=38.7GiB | 63.7% done | ETA(epoch): 1075s + [Epoch 191][20/66] loss=0.934383 avg=0.861130 VRAM=38.6GiB | 63.8% done | ETA(epoch): 883s + [Epoch 191][30/66] loss=0.735237 avg=0.852043 VRAM=38.7GiB | 63.8% done | ETA(epoch): 691s + [Epoch 191][40/66] loss=0.763479 avg=0.853064 VRAM=38.6GiB | 63.9% done | ETA(epoch): 499s + [Epoch 191][50/66] loss=0.942147 avg=0.849328 VRAM=38.7GiB | 63.9% done | ETA(epoch): 307s + [Epoch 191][60/66] loss=0.940411 avg=0.850256 VRAM=38.6GiB | 64.0% done | ETA(epoch): 115s + Train loss: 0.853962 (1266.3s) ETA: 2524min + Val loss: 0.876436 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0075 t_0.4-0.6=0.8292 t_0.6-0.8=0.6918 t_0.8-1.0=0.6269] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0191 + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 192/299 (64% done) --- + [Epoch 192][10/66] loss=0.942936 avg=0.866908 VRAM=38.7GiB | 64.1% done | ETA(epoch): 1074s + [Epoch 192][20/66] loss=0.880847 avg=0.868224 VRAM=38.6GiB | 64.1% done | ETA(epoch): 882s + [Epoch 192][30/66] loss=0.800312 avg=0.873400 VRAM=38.7GiB | 64.2% done | ETA(epoch): 690s + [Epoch 192][40/66] loss=0.719106 avg=0.859878 VRAM=38.6GiB | 64.2% done | ETA(epoch): 500s + [Epoch 192][50/66] loss=0.973844 avg=0.858108 VRAM=38.7GiB | 64.3% done | ETA(epoch): 307s + [Epoch 192][60/66] loss=0.823095 avg=0.854967 VRAM=38.6GiB | 64.3% done | ETA(epoch): 115s + Train loss: 0.852279 (1267.9s) ETA: 2501min + Val loss: 0.853267 [t_0.0-0.2=1.0740 t_0.2-0.4=1.0055 t_0.4-0.6=0.8535 t_0.6-0.8=0.7025 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 193/299 (64% done) --- + [Epoch 193][10/66] loss=0.854035 avg=0.853213 VRAM=38.7GiB | 64.4% done | ETA(epoch): 1074s + [Epoch 193][20/66] loss=0.810210 avg=0.855237 VRAM=38.6GiB | 64.4% done | ETA(epoch): 883s + [Epoch 193][30/66] loss=0.729671 avg=0.852184 VRAM=38.7GiB | 64.5% done | ETA(epoch): 691s + [Epoch 193][40/66] loss=0.910457 avg=0.856402 VRAM=38.6GiB | 64.5% done | ETA(epoch): 499s + [Epoch 193][50/66] loss=0.941680 avg=0.857225 VRAM=38.7GiB | 64.6% done | ETA(epoch): 307s + [Epoch 193][60/66] loss=0.736962 avg=0.856243 VRAM=38.6GiB | 64.6% done | ETA(epoch): 115s + Train loss: 0.852434 (1266.0s) ETA: 2477min + Val loss: 0.839807 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9733 t_0.4-0.6=0.8272 t_0.6-0.8=0.7428 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 194/299 (65% done) --- + [Epoch 194][10/66] loss=0.857321 avg=0.838691 VRAM=38.7GiB | 64.7% done | ETA(epoch): 1072s + [Epoch 194][20/66] loss=0.871474 avg=0.838632 VRAM=38.6GiB | 64.8% done | ETA(epoch): 880s + [Epoch 194][30/66] loss=0.915295 avg=0.834713 VRAM=38.7GiB | 64.8% done | ETA(epoch): 689s + [Epoch 194][40/66] loss=0.869480 avg=0.852424 VRAM=38.6GiB | 64.9% done | ETA(epoch): 498s + [Epoch 194][50/66] loss=0.880450 avg=0.852692 VRAM=38.7GiB | 64.9% done | ETA(epoch): 306s + [Epoch 194][60/66] loss=0.766019 avg=0.847460 VRAM=38.6GiB | 65.0% done | ETA(epoch): 115s + Train loss: 0.848783 (1263.0s) ETA: 2453min + Val loss: 0.882964 [t_0.0-0.2=1.0704 t_0.2-0.4=0.9763 t_0.4-0.6=0.8813 t_0.6-0.8=0.7075 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 195/299 (65% done) --- + [Epoch 195][10/66] loss=0.976595 avg=0.865277 VRAM=38.7GiB | 65.1% done | ETA(epoch): 1090s + [Epoch 195][20/66] loss=0.797580 avg=0.848894 VRAM=38.6GiB | 65.1% done | ETA(epoch): 889s + [Epoch 195][30/66] loss=0.920743 avg=0.857703 VRAM=38.7GiB | 65.2% done | ETA(epoch): 694s + [Epoch 195][40/66] loss=0.910621 avg=0.856450 VRAM=38.6GiB | 65.2% done | ETA(epoch): 501s + [Epoch 195][50/66] loss=0.879854 avg=0.851373 VRAM=38.7GiB | 65.3% done | ETA(epoch): 308s + [Epoch 195][60/66] loss=0.850001 avg=0.852354 VRAM=38.6GiB | 65.3% done | ETA(epoch): 115s + Train loss: 0.852183 (1268.6s) ETA: 2429min + Val loss: 0.850121 [t_0.0-0.2=1.0796 t_0.2-0.4=1.0234 t_0.4-0.6=0.8563 t_0.6-0.8=0.6997 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 196/299 (65% done) --- + [Epoch 196][10/66] loss=0.817721 avg=0.856796 VRAM=38.7GiB | 65.4% done | ETA(epoch): 1091s + [Epoch 196][20/66] loss=0.835157 avg=0.868538 VRAM=38.6GiB | 65.4% done | ETA(epoch): 889s + [Epoch 196][30/66] loss=0.959156 avg=0.861078 VRAM=38.7GiB | 65.5% done | ETA(epoch): 694s + [Epoch 196][40/66] loss=0.792726 avg=0.852782 VRAM=38.6GiB | 65.5% done | ETA(epoch): 501s + [Epoch 196][50/66] loss=0.850513 avg=0.854736 VRAM=38.7GiB | 65.6% done | ETA(epoch): 308s + [Epoch 196][60/66] loss=0.924966 avg=0.857230 VRAM=38.6GiB | 65.6% done | ETA(epoch): 115s + Train loss: 0.854303 (1269.9s) ETA: 2406min + Val loss: 0.856628 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9950 t_0.4-0.6=0.8278 t_0.6-0.8=0.7271 t_0.8-1.0=0.6160] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 197/299 (66% done) --- + [Epoch 197][10/66] loss=0.850011 avg=0.856264 VRAM=38.7GiB | 65.7% done | ETA(epoch): 1074s + [Epoch 197][20/66] loss=0.765106 avg=0.849821 VRAM=38.6GiB | 65.8% done | ETA(epoch): 882s + [Epoch 197][30/66] loss=0.827844 avg=0.826369 VRAM=38.7GiB | 65.8% done | ETA(epoch): 691s + [Epoch 197][40/66] loss=0.888337 avg=0.833136 VRAM=38.6GiB | 65.9% done | ETA(epoch): 499s + [Epoch 197][50/66] loss=0.748352 avg=0.827800 VRAM=38.7GiB | 65.9% done | ETA(epoch): 307s + [Epoch 197][60/66] loss=0.848038 avg=0.835701 VRAM=38.6GiB | 66.0% done | ETA(epoch): 115s + Train loss: 0.833475 (1268.5s) ETA: 2382min + Val loss: 0.856617 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9716 t_0.4-0.6=0.8491 t_0.6-0.8=0.6902 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 198/299 (66% done) --- + [Epoch 198][10/66] loss=0.792013 avg=0.856878 VRAM=38.7GiB | 66.1% done | ETA(epoch): 1076s + [Epoch 198][20/66] loss=0.919245 avg=0.864002 VRAM=38.6GiB | 66.1% done | ETA(epoch): 883s + [Epoch 198][30/66] loss=0.797367 avg=0.863282 VRAM=38.7GiB | 66.2% done | ETA(epoch): 691s + [Epoch 198][40/66] loss=0.832343 avg=0.858324 VRAM=38.6GiB | 66.2% done | ETA(epoch): 499s + [Epoch 198][50/66] loss=0.899493 avg=0.861665 VRAM=38.7GiB | 66.3% done | ETA(epoch): 308s + [Epoch 198][60/66] loss=0.798831 avg=0.860998 VRAM=38.6GiB | 66.3% done | ETA(epoch): 115s + Train loss: 0.855963 (1268.1s) ETA: 2358min + Val loss: 0.852759 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9697 t_0.4-0.6=0.8746 t_0.6-0.8=0.6938 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 199/299 (66% done) --- + [Epoch 199][10/66] loss=0.870542 avg=0.818696 VRAM=38.7GiB | 66.4% done | ETA(epoch): 1073s + [Epoch 199][20/66] loss=0.879546 avg=0.828900 VRAM=38.6GiB | 66.4% done | ETA(epoch): 881s + [Epoch 199][30/66] loss=0.989457 avg=0.842081 VRAM=38.7GiB | 66.5% done | ETA(epoch): 690s + [Epoch 199][40/66] loss=0.987369 avg=0.853122 VRAM=38.6GiB | 66.5% done | ETA(epoch): 500s + [Epoch 199][50/66] loss=0.727128 avg=0.851678 VRAM=38.7GiB | 66.6% done | ETA(epoch): 308s + [Epoch 199][60/66] loss=0.978617 avg=0.854148 VRAM=38.6GiB | 66.6% done | ETA(epoch): 115s + Train loss: 0.851293 (1268.7s) ETA: 2335min + Val loss: 0.836835 [t_0.0-0.2=1.0741 t_0.2-0.4=0.9730 t_0.4-0.6=0.8325 t_0.6-0.8=0.7164 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 200/299 (67% done) --- + [Epoch 200][10/66] loss=0.940737 avg=0.867821 VRAM=38.7GiB | 66.7% done | ETA(epoch): 1074s + [Epoch 200][20/66] loss=0.757518 avg=0.853507 VRAM=38.6GiB | 66.8% done | ETA(epoch): 882s + [Epoch 200][30/66] loss=0.922006 avg=0.861147 VRAM=38.7GiB | 66.8% done | ETA(epoch): 690s + [Epoch 200][40/66] loss=0.837904 avg=0.859962 VRAM=38.6GiB | 66.9% done | ETA(epoch): 501s + [Epoch 200][50/66] loss=1.012298 avg=0.857599 VRAM=38.7GiB | 66.9% done | ETA(epoch): 308s + [Epoch 200][60/66] loss=0.943400 avg=0.860150 VRAM=38.6GiB | 67.0% done | ETA(epoch): 115s + Train loss: 0.860687 (1269.3s) ETA: 2311min + Val loss: 0.827709 [t_0.0-0.2=1.0542 t_0.2-0.4=1.0066 t_0.4-0.6=0.8142 t_0.6-0.8=0.7261 t_0.8-1.0=0.6577] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 201/299 (67% done) --- + [Epoch 201][10/66] loss=0.917679 avg=0.827348 VRAM=38.7GiB | 67.1% done | ETA(epoch): 1075s + [Epoch 201][20/66] loss=0.934503 avg=0.850754 VRAM=38.6GiB | 67.1% done | ETA(epoch): 882s + [Epoch 201][30/66] loss=0.826160 avg=0.846311 VRAM=38.7GiB | 67.2% done | ETA(epoch): 691s + [Epoch 201][40/66] loss=0.958904 avg=0.835664 VRAM=38.6GiB | 67.2% done | ETA(epoch): 499s + [Epoch 201][50/66] loss=0.966722 avg=0.841729 VRAM=38.7GiB | 67.3% done | ETA(epoch): 307s + [Epoch 201][60/66] loss=0.832263 avg=0.844027 VRAM=38.6GiB | 67.3% done | ETA(epoch): 115s + Train loss: 0.841237 (1265.9s) ETA: 2287min + Val loss: 0.881774 [t_0.0-0.2=1.0772 t_0.2-0.4=0.9506 t_0.4-0.6=0.8871 t_0.6-0.8=0.7135 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 202/299 (67% done) --- + [Epoch 202][10/66] loss=0.846713 avg=0.857609 VRAM=38.7GiB | 67.4% done | ETA(epoch): 1075s + [Epoch 202][20/66] loss=0.974806 avg=0.855362 VRAM=38.6GiB | 67.4% done | ETA(epoch): 891s + [Epoch 202][30/66] loss=0.827211 avg=0.862591 VRAM=38.7GiB | 67.5% done | ETA(epoch): 696s + [Epoch 202][40/66] loss=0.866772 avg=0.851337 VRAM=38.6GiB | 67.5% done | ETA(epoch): 502s + [Epoch 202][50/66] loss=0.783002 avg=0.847979 VRAM=38.7GiB | 67.6% done | ETA(epoch): 308s + [Epoch 202][60/66] loss=0.793469 avg=0.848029 VRAM=38.6GiB | 67.6% done | ETA(epoch): 116s + Train loss: 0.847008 (1270.2s) ETA: 2264min + Val loss: 0.846109 [t_0.0-0.2=1.0603 t_0.2-0.4=1.0119 t_0.4-0.6=0.8309 t_0.6-0.8=0.7037 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 203/299 (68% done) --- + [Epoch 203][10/66] loss=0.888561 avg=0.818305 VRAM=38.7GiB | 67.7% done | ETA(epoch): 1087s + [Epoch 203][20/66] loss=0.871690 avg=0.846786 VRAM=38.6GiB | 67.8% done | ETA(epoch): 888s + [Epoch 203][30/66] loss=1.017450 avg=0.851136 VRAM=38.7GiB | 67.8% done | ETA(epoch): 693s + [Epoch 203][40/66] loss=0.869884 avg=0.843817 VRAM=38.6GiB | 67.9% done | ETA(epoch): 500s + [Epoch 203][50/66] loss=0.897332 avg=0.850633 VRAM=38.7GiB | 67.9% done | ETA(epoch): 308s + [Epoch 203][60/66] loss=0.909750 avg=0.852391 VRAM=38.6GiB | 68.0% done | ETA(epoch): 115s + Train loss: 0.854593 (1269.2s) ETA: 2240min + Val loss: 0.842244 [t_0.0-0.2=1.0818 t_0.2-0.4=1.0111 t_0.4-0.6=0.8570 t_0.6-0.8=0.7207 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 204/299 (68% done) --- + [Epoch 204][10/66] loss=0.818883 avg=0.880218 VRAM=38.7GiB | 68.1% done | ETA(epoch): 1074s + [Epoch 204][20/66] loss=0.809494 avg=0.863166 VRAM=38.6GiB | 68.1% done | ETA(epoch): 883s + [Epoch 204][30/66] loss=0.941991 avg=0.850969 VRAM=38.7GiB | 68.2% done | ETA(epoch): 691s + [Epoch 204][40/66] loss=0.797612 avg=0.849464 VRAM=38.6GiB | 68.2% done | ETA(epoch): 499s + [Epoch 204][50/66] loss=0.900030 avg=0.844634 VRAM=38.7GiB | 68.3% done | ETA(epoch): 307s + [Epoch 204][60/66] loss=0.778445 avg=0.842480 VRAM=38.6GiB | 68.3% done | ETA(epoch): 115s + Train loss: 0.843773 (1270.1s) ETA: 2216min + Val loss: 0.856728 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9658 t_0.4-0.6=0.8522 t_0.6-0.8=0.7374 t_0.8-1.0=0.6553] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 205/299 (68% done) --- + [Epoch 205][10/66] loss=0.839635 avg=0.879388 VRAM=38.7GiB | 68.4% done | ETA(epoch): 1074s + [Epoch 205][20/66] loss=0.799830 avg=0.871826 VRAM=38.6GiB | 68.4% done | ETA(epoch): 882s + [Epoch 205][30/66] loss=0.806481 avg=0.868334 VRAM=38.7GiB | 68.5% done | ETA(epoch): 691s + [Epoch 205][40/66] loss=0.851120 avg=0.860774 VRAM=38.6GiB | 68.5% done | ETA(epoch): 499s + [Epoch 205][50/66] loss=0.957978 avg=0.857524 VRAM=38.7GiB | 68.6% done | ETA(epoch): 307s + [Epoch 205][60/66] loss=0.697318 avg=0.848465 VRAM=38.6GiB | 68.6% done | ETA(epoch): 115s + Train loss: 0.848581 (1269.5s) ETA: 2193min + Val loss: 0.827942 [t_0.0-0.2=1.0705 t_0.2-0.4=0.9979 t_0.4-0.6=0.8471 t_0.6-0.8=0.6941 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 206/299 (69% done) --- + [Epoch 206][10/66] loss=0.817351 avg=0.893958 VRAM=38.7GiB | 68.7% done | ETA(epoch): 1073s + [Epoch 206][20/66] loss=0.804770 avg=0.856757 VRAM=38.6GiB | 68.8% done | ETA(epoch): 882s + [Epoch 206][30/66] loss=0.823095 avg=0.858814 VRAM=38.7GiB | 68.8% done | ETA(epoch): 690s + [Epoch 206][40/66] loss=0.961591 avg=0.855422 VRAM=38.6GiB | 68.9% done | ETA(epoch): 499s + [Epoch 206][50/66] loss=0.811987 avg=0.849151 VRAM=38.7GiB | 68.9% done | ETA(epoch): 307s + [Epoch 206][60/66] loss=0.842024 avg=0.847396 VRAM=38.6GiB | 69.0% done | ETA(epoch): 115s + Train loss: 0.849076 (1268.4s) ETA: 2169min + Val loss: 0.848452 [t_0.0-0.2=1.0570 t_0.2-0.4=1.0004 t_0.4-0.6=0.8672 t_0.6-0.8=0.7093 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 207/299 (69% done) --- + [Epoch 207][10/66] loss=0.862700 avg=0.825904 VRAM=38.7GiB | 69.1% done | ETA(epoch): 1074s + [Epoch 207][20/66] loss=0.971308 avg=0.837192 VRAM=38.6GiB | 69.1% done | ETA(epoch): 882s + [Epoch 207][30/66] loss=0.763184 avg=0.843987 VRAM=38.7GiB | 69.2% done | ETA(epoch): 691s + [Epoch 207][40/66] loss=0.714846 avg=0.844906 VRAM=38.6GiB | 69.2% done | ETA(epoch): 499s + [Epoch 207][50/66] loss=0.951911 avg=0.847004 VRAM=38.7GiB | 69.3% done | ETA(epoch): 307s + [Epoch 207][60/66] loss=1.007057 avg=0.851416 VRAM=38.6GiB | 69.3% done | ETA(epoch): 115s + Train loss: 0.851455 (1266.2s) ETA: 2146min + Val loss: 0.836491 [t_0.0-0.2=1.0591 t_0.2-0.4=0.9784 t_0.4-0.6=0.8362 t_0.6-0.8=0.7328 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 208/299 (69% done) --- + [Epoch 208][10/66] loss=0.757496 avg=0.803645 VRAM=38.7GiB | 69.4% done | ETA(epoch): 1073s + [Epoch 208][20/66] loss=0.805242 avg=0.836220 VRAM=38.6GiB | 69.4% done | ETA(epoch): 882s + [Epoch 208][30/66] loss=0.890335 avg=0.836972 VRAM=38.7GiB | 69.5% done | ETA(epoch): 690s + [Epoch 208][40/66] loss=0.870397 avg=0.841720 VRAM=38.6GiB | 69.5% done | ETA(epoch): 499s + [Epoch 208][50/66] loss=0.827336 avg=0.847592 VRAM=38.7GiB | 69.6% done | ETA(epoch): 307s + [Epoch 208][60/66] loss=0.862135 avg=0.849299 VRAM=38.6GiB | 69.6% done | ETA(epoch): 115s + Train loss: 0.848579 (1266.3s) ETA: 2122min + Val loss: 0.863054 [t_0.0-0.2=1.0827 t_0.2-0.4=0.9933 t_0.4-0.6=0.8287 t_0.6-0.8=0.6960 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 209/299 (70% done) --- + [Epoch 209][10/66] loss=0.773383 avg=0.855953 VRAM=38.7GiB | 69.7% done | ETA(epoch): 1073s + [Epoch 209][20/66] loss=0.945947 avg=0.860833 VRAM=38.6GiB | 69.8% done | ETA(epoch): 883s + [Epoch 209][30/66] loss=0.937342 avg=0.854641 VRAM=38.7GiB | 69.8% done | ETA(epoch): 693s + [Epoch 209][40/66] loss=0.881077 avg=0.854668 VRAM=38.6GiB | 69.9% done | ETA(epoch): 500s + [Epoch 209][50/66] loss=0.855349 avg=0.854865 VRAM=38.7GiB | 69.9% done | ETA(epoch): 308s + [Epoch 209][60/66] loss=0.974665 avg=0.857801 VRAM=38.6GiB | 70.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 13859 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 13859 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 13859 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 13859 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 13859 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 13859 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 13859 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 13859 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 13859 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 13859 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.855163 (2303.6s) ETA: 2106min + Val loss: 0.802968 [t_0.0-0.2=1.0549 t_0.2-0.4=0.9751 t_0.4-0.6=0.8498 t_0.6-0.8=0.7158 t_0.8-1.0=0.6357] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0209 (BEST) + Deleted old checkpoint: checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 210/299 (70% done) --- + [Epoch 210][10/66] loss=0.723461 avg=0.811471 VRAM=38.7GiB | 70.1% done | ETA(epoch): 1075s + [Epoch 210][20/66] loss=0.879192 avg=0.839162 VRAM=38.6GiB | 70.1% done | ETA(epoch): 883s + [Epoch 210][30/66] loss=0.803020 avg=0.857135 VRAM=38.7GiB | 70.2% done | ETA(epoch): 691s + [Epoch 210][40/66] loss=0.689816 avg=0.856966 VRAM=38.6GiB | 70.2% done | ETA(epoch): 499s + [Epoch 210][50/66] loss=0.863586 avg=0.854138 VRAM=38.7GiB | 70.3% done | ETA(epoch): 307s + [Epoch 210][60/66] loss=0.926042 avg=0.855275 VRAM=38.6GiB | 70.3% done | ETA(epoch): 115s + Train loss: 0.853658 (1267.1s) ETA: 2082min + Val loss: 0.886654 [t_0.0-0.2=1.0728 t_0.2-0.4=1.0028 t_0.4-0.6=0.8743 t_0.6-0.8=0.7188 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 211/299 (70% done) --- + [Epoch 211][10/66] loss=0.863975 avg=0.852634 VRAM=38.7GiB | 70.4% done | ETA(epoch): 1075s + [Epoch 211][20/66] loss=0.839550 avg=0.872528 VRAM=38.6GiB | 70.4% done | ETA(epoch): 888s + [Epoch 211][30/66] loss=0.839218 avg=0.865268 VRAM=38.7GiB | 70.5% done | ETA(epoch): 694s + [Epoch 211][40/66] loss=0.963507 avg=0.867270 VRAM=38.6GiB | 70.5% done | ETA(epoch): 501s + [Epoch 211][50/66] loss=0.805740 avg=0.858344 VRAM=38.7GiB | 70.6% done | ETA(epoch): 308s + [Epoch 211][60/66] loss=0.834804 avg=0.857171 VRAM=38.6GiB | 70.6% done | ETA(epoch): 115s + Train loss: 0.854999 (1269.4s) ETA: 2058min + Val loss: 0.830899 [t_0.0-0.2=1.0860 t_0.2-0.4=0.9993 t_0.4-0.6=0.8505 t_0.6-0.8=0.7127 t_0.8-1.0=0.6346] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 212/299 (71% done) --- + [Epoch 212][10/66] loss=0.841916 avg=0.875280 VRAM=38.7GiB | 70.7% done | ETA(epoch): 1087s + [Epoch 212][20/66] loss=0.754300 avg=0.855688 VRAM=38.6GiB | 70.8% done | ETA(epoch): 888s + [Epoch 212][30/66] loss=0.885814 avg=0.858668 VRAM=38.7GiB | 70.8% done | ETA(epoch): 694s + [Epoch 212][40/66] loss=0.940682 avg=0.860434 VRAM=38.6GiB | 70.9% done | ETA(epoch): 501s + [Epoch 212][50/66] loss=0.902568 avg=0.859656 VRAM=38.7GiB | 70.9% done | ETA(epoch): 308s + [Epoch 212][60/66] loss=0.893266 avg=0.860161 VRAM=38.6GiB | 71.0% done | ETA(epoch): 115s + Train loss: 0.862692 (1269.9s) ETA: 2035min + Val loss: 0.817742 [t_0.0-0.2=1.0655 t_0.2-0.4=1.0114 t_0.4-0.6=0.8312 t_0.6-0.8=0.7227 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0212 +[MEM @ epoch 212 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 213/299 (71% done) --- + [Epoch 213][10/66] loss=0.904491 avg=0.874239 VRAM=38.7GiB | 71.1% done | ETA(epoch): 1075s + [Epoch 213][20/66] loss=0.877058 avg=0.873949 VRAM=38.6GiB | 71.1% done | ETA(epoch): 885s + [Epoch 213][30/66] loss=0.833937 avg=0.852201 VRAM=38.7GiB | 71.2% done | ETA(epoch): 692s + [Epoch 213][40/66] loss=0.954039 avg=0.859479 VRAM=38.6GiB | 71.2% done | ETA(epoch): 500s + [Epoch 213][50/66] loss=0.962160 avg=0.849794 VRAM=38.7GiB | 71.3% done | ETA(epoch): 307s + [Epoch 213][60/66] loss=0.794681 avg=0.852278 VRAM=38.6GiB | 71.3% done | ETA(epoch): 116s + Train loss: 0.852179 (1270.4s) ETA: 2011min + Val loss: 0.856035 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9478 t_0.4-0.6=0.8619 t_0.6-0.8=0.7251 t_0.8-1.0=0.6116] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 20.8/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 214/299 (71% done) --- + [Epoch 214][10/66] loss=0.914242 avg=0.851019 VRAM=38.7GiB | 71.4% done | ETA(epoch): 1074s + [Epoch 214][20/66] loss=0.783427 avg=0.846146 VRAM=38.6GiB | 71.4% done | ETA(epoch): 883s + [Epoch 214][30/66] loss=0.816677 avg=0.846374 VRAM=38.7GiB | 71.5% done | ETA(epoch): 691s + [Epoch 214][40/66] loss=0.977450 avg=0.843271 VRAM=38.6GiB | 71.5% done | ETA(epoch): 499s + [Epoch 214][50/66] loss=0.946628 avg=0.846449 VRAM=38.7GiB | 71.6% done | ETA(epoch): 307s + [Epoch 214][60/66] loss=0.875786 avg=0.850071 VRAM=38.6GiB | 71.6% done | ETA(epoch): 115s + Train loss: 0.849905 (1269.3s) ETA: 1988min + Val loss: 0.839733 [t_0.0-0.2=1.0616 t_0.2-0.4=1.0032 t_0.4-0.6=0.8419 t_0.6-0.8=0.7084 t_0.8-1.0=0.6441] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 215/299 (72% done) --- + [Epoch 215][10/66] loss=0.876003 avg=0.853551 VRAM=38.7GiB | 71.7% done | ETA(epoch): 1074s + [Epoch 215][20/66] loss=0.968427 avg=0.838325 VRAM=38.6GiB | 71.8% done | ETA(epoch): 883s + [Epoch 215][30/66] loss=0.951936 avg=0.836679 VRAM=38.7GiB | 71.8% done | ETA(epoch): 691s + [Epoch 215][40/66] loss=0.764468 avg=0.836808 VRAM=38.6GiB | 71.9% done | ETA(epoch): 499s + [Epoch 215][50/66] loss=0.880802 avg=0.833441 VRAM=38.7GiB | 71.9% done | ETA(epoch): 308s + [Epoch 215][60/66] loss=0.866689 avg=0.827644 VRAM=38.6GiB | 72.0% done | ETA(epoch): 115s + Train loss: 0.827860 (1269.5s) ETA: 1964min + Val loss: 0.854590 [t_0.0-0.2=1.0850 t_0.2-0.4=0.9785 t_0.4-0.6=0.8255 t_0.6-0.8=0.7304 t_0.8-1.0=0.6317] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 216/299 (72% done) --- + [Epoch 216][10/66] loss=0.891238 avg=0.857927 VRAM=38.7GiB | 72.1% done | ETA(epoch): 1077s + [Epoch 216][20/66] loss=0.802194 avg=0.856782 VRAM=38.6GiB | 72.1% done | ETA(epoch): 884s + [Epoch 216][30/66] loss=0.804121 avg=0.858314 VRAM=38.7GiB | 72.2% done | ETA(epoch): 692s + [Epoch 216][40/66] loss=0.928009 avg=0.853790 VRAM=38.6GiB | 72.2% done | ETA(epoch): 500s + [Epoch 216][50/66] loss=0.910845 avg=0.860744 VRAM=38.7GiB | 72.3% done | ETA(epoch): 308s + [Epoch 216][60/66] loss=0.846038 avg=0.863004 VRAM=38.6GiB | 72.3% done | ETA(epoch): 115s + Train loss: 0.863290 (1269.0s) ETA: 1940min + Val loss: 0.856141 [t_0.0-0.2=1.0725 t_0.2-0.4=0.9583 t_0.4-0.6=0.8421 t_0.6-0.8=0.6913 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 21.2/188.5 GiB (11.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 217/299 (72% done) --- + [Epoch 217][10/66] loss=0.926086 avg=0.871661 VRAM=38.7GiB | 72.4% done | ETA(epoch): 1075s + [Epoch 217][20/66] loss=0.773343 avg=0.869382 VRAM=38.6GiB | 72.4% done | ETA(epoch): 884s + [Epoch 217][30/66] loss=0.833142 avg=0.859815 VRAM=38.7GiB | 72.5% done | ETA(epoch): 692s + [Epoch 217][40/66] loss=0.787012 avg=0.858573 VRAM=38.6GiB | 72.5% done | ETA(epoch): 500s + [Epoch 217][50/66] loss=0.874145 avg=0.852216 VRAM=38.7GiB | 72.6% done | ETA(epoch): 307s + [Epoch 217][60/66] loss=0.760984 avg=0.851975 VRAM=38.6GiB | 72.6% done | ETA(epoch): 115s + Train loss: 0.854328 (1267.8s) ETA: 1917min + Val loss: 0.853327 [t_0.0-0.2=1.0623 t_0.2-0.4=1.0045 t_0.4-0.6=0.8459 t_0.6-0.8=0.7247 t_0.8-1.0=0.6355] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 218/299 (73% done) --- + [Epoch 218][10/66] loss=0.828719 avg=0.863748 VRAM=38.7GiB | 72.7% done | ETA(epoch): 1076s + [Epoch 218][20/66] loss=0.720785 avg=0.867195 VRAM=38.6GiB | 72.8% done | ETA(epoch): 888s + [Epoch 218][30/66] loss=0.747444 avg=0.855165 VRAM=38.7GiB | 72.8% done | ETA(epoch): 694s + [Epoch 218][40/66] loss=0.884584 avg=0.847479 VRAM=38.6GiB | 72.9% done | ETA(epoch): 501s + [Epoch 218][50/66] loss=0.946486 avg=0.852790 VRAM=38.7GiB | 72.9% done | ETA(epoch): 308s + [Epoch 218][60/66] loss=0.960174 avg=0.854254 VRAM=38.6GiB | 73.0% done | ETA(epoch): 115s + Train loss: 0.848607 (1269.5s) ETA: 1893min + Val loss: 0.845246 [t_0.0-0.2=1.0523 t_0.2-0.4=1.0028 t_0.4-0.6=0.8571 t_0.6-0.8=0.7188 t_0.8-1.0=0.6449] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 219/299 (73% done) --- + [Epoch 219][10/66] loss=0.812518 avg=0.862197 VRAM=38.7GiB | 73.1% done | ETA(epoch): 1087s + [Epoch 219][20/66] loss=0.955534 avg=0.866655 VRAM=38.6GiB | 73.1% done | ETA(epoch): 889s + [Epoch 219][30/66] loss=0.934166 avg=0.867482 VRAM=38.7GiB | 73.2% done | ETA(epoch): 694s + [Epoch 219][40/66] loss=0.768605 avg=0.866697 VRAM=38.6GiB | 73.2% done | ETA(epoch): 501s + [Epoch 219][50/66] loss=0.795234 avg=0.858025 VRAM=38.7GiB | 73.3% done | ETA(epoch): 308s + [Epoch 219][60/66] loss=0.905299 avg=0.855968 VRAM=38.6GiB | 73.3% done | ETA(epoch): 116s + Train loss: 0.858668 (1270.6s) ETA: 1869min + Val loss: 0.863473 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9942 t_0.4-0.6=0.8460 t_0.6-0.8=0.7019 t_0.8-1.0=0.6277] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 220/299 (73% done) --- + [Epoch 220][10/66] loss=0.772840 avg=0.799947 VRAM=38.7GiB | 73.4% done | ETA(epoch): 1087s + [Epoch 220][20/66] loss=0.710002 avg=0.825332 VRAM=38.6GiB | 73.4% done | ETA(epoch): 888s + [Epoch 220][30/66] loss=0.799463 avg=0.842530 VRAM=38.7GiB | 73.5% done | ETA(epoch): 693s + [Epoch 220][40/66] loss=0.961417 avg=0.846339 VRAM=38.6GiB | 73.5% done | ETA(epoch): 501s + [Epoch 220][50/66] loss=0.844929 avg=0.843276 VRAM=38.7GiB | 73.6% done | ETA(epoch): 308s + [Epoch 220][60/66] loss=0.968314 avg=0.851240 VRAM=38.6GiB | 73.6% done | ETA(epoch): 115s + Train loss: 0.854337 (1269.4s) ETA: 1846min + Val loss: 0.869582 [t_0.0-0.2=1.0635 t_0.2-0.4=1.0066 t_0.4-0.6=0.8627 t_0.6-0.8=0.7070 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 221/299 (74% done) --- + [Epoch 221][10/66] loss=0.849275 avg=0.829038 VRAM=38.7GiB | 73.7% done | ETA(epoch): 1078s + [Epoch 221][20/66] loss=0.876973 avg=0.858476 VRAM=38.6GiB | 73.8% done | ETA(epoch): 885s + [Epoch 221][30/66] loss=0.944091 avg=0.852264 VRAM=38.7GiB | 73.8% done | ETA(epoch): 692s + [Epoch 221][40/66] loss=0.833249 avg=0.851686 VRAM=38.6GiB | 73.9% done | ETA(epoch): 500s + [Epoch 221][50/66] loss=0.946993 avg=0.855521 VRAM=38.7GiB | 73.9% done | ETA(epoch): 307s + [Epoch 221][60/66] loss=0.950373 avg=0.859405 VRAM=38.6GiB | 74.0% done | ETA(epoch): 115s + Train loss: 0.855287 (1268.1s) ETA: 1822min + Val loss: 0.856875 [t_0.0-0.2=1.0605 t_0.2-0.4=1.0096 t_0.4-0.6=0.8495 t_0.6-0.8=0.7128 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 222/299 (74% done) --- + [Epoch 222][10/66] loss=0.751154 avg=0.859199 VRAM=38.7GiB | 74.1% done | ETA(epoch): 1075s + [Epoch 222][20/66] loss=0.971941 avg=0.832872 VRAM=38.6GiB | 74.1% done | ETA(epoch): 884s + [Epoch 222][30/66] loss=0.841418 avg=0.838430 VRAM=38.7GiB | 74.2% done | ETA(epoch): 692s + [Epoch 222][40/66] loss=0.841529 avg=0.845049 VRAM=38.6GiB | 74.2% done | ETA(epoch): 500s + [Epoch 222][50/66] loss=0.834353 avg=0.851338 VRAM=38.7GiB | 74.3% done | ETA(epoch): 308s + [Epoch 222][60/66] loss=0.790221 avg=0.854161 VRAM=38.6GiB | 74.3% done | ETA(epoch): 115s + Train loss: 0.850268 (1270.2s) ETA: 1799min + Val loss: 0.878712 [t_0.0-0.2=1.0625 t_0.2-0.4=0.9961 t_0.4-0.6=0.8605 t_0.6-0.8=0.7183 t_0.8-1.0=0.6298] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 223/299 (74% done) --- + [Epoch 223][10/66] loss=0.861197 avg=0.907084 VRAM=38.7GiB | 74.4% done | ETA(epoch): 1075s + [Epoch 223][20/66] loss=0.901859 avg=0.896382 VRAM=38.6GiB | 74.4% done | ETA(epoch): 882s + [Epoch 223][30/66] loss=0.819861 avg=0.882160 VRAM=38.7GiB | 74.5% done | ETA(epoch): 691s + [Epoch 223][40/66] loss=0.891597 avg=0.877477 VRAM=38.6GiB | 74.5% done | ETA(epoch): 502s + [Epoch 223][50/66] loss=0.820897 avg=0.878603 VRAM=38.7GiB | 74.6% done | ETA(epoch): 309s + [Epoch 223][60/66] loss=0.792737 avg=0.877743 VRAM=38.6GiB | 74.6% done | ETA(epoch): 116s + Train loss: 0.867464 (1272.2s) ETA: 1775min + Val loss: 0.859329 [t_0.0-0.2=1.0850 t_0.2-0.4=1.0056 t_0.4-0.6=0.8484 t_0.6-0.8=0.7152 t_0.8-1.0=0.6388] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 224/299 (75% done) --- + [Epoch 224][10/66] loss=0.875063 avg=0.881534 VRAM=38.7GiB | 74.7% done | ETA(epoch): 1078s + [Epoch 224][20/66] loss=1.012748 avg=0.857497 VRAM=38.6GiB | 74.8% done | ETA(epoch): 884s + [Epoch 224][30/66] loss=0.795699 avg=0.866987 VRAM=38.7GiB | 74.8% done | ETA(epoch): 692s + [Epoch 224][40/66] loss=0.842881 avg=0.865632 VRAM=38.6GiB | 74.9% done | ETA(epoch): 500s + [Epoch 224][50/66] loss=0.856408 avg=0.869074 VRAM=38.7GiB | 74.9% done | ETA(epoch): 307s + [Epoch 224][60/66] loss=0.758143 avg=0.868105 VRAM=38.6GiB | 75.0% done | ETA(epoch): 115s + Train loss: 0.866111 (1267.6s) ETA: 1752min + Val loss: 0.875421 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0008 t_0.4-0.6=0.8171 t_0.6-0.8=0.7129 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 225/299 (75% done) --- + [Epoch 225][10/66] loss=0.904849 avg=0.860473 VRAM=38.7GiB | 75.1% done | ETA(epoch): 1076s + [Epoch 225][20/66] loss=0.810630 avg=0.851964 VRAM=38.6GiB | 75.1% done | ETA(epoch): 883s + [Epoch 225][30/66] loss=0.844885 avg=0.852664 VRAM=38.7GiB | 75.2% done | ETA(epoch): 693s + [Epoch 225][40/66] loss=0.928674 avg=0.856296 VRAM=38.6GiB | 75.2% done | ETA(epoch): 500s + [Epoch 225][50/66] loss=0.927149 avg=0.856311 VRAM=38.7GiB | 75.3% done | ETA(epoch): 308s + [Epoch 225][60/66] loss=0.813589 avg=0.852638 VRAM=38.6GiB | 75.3% done | ETA(epoch): 115s + Train loss: 0.851644 (1268.9s) ETA: 1728min + Val loss: 0.840958 [t_0.0-0.2=1.0655 t_0.2-0.4=1.0022 t_0.4-0.6=0.8684 t_0.6-0.8=0.7015 t_0.8-1.0=0.6308] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 226/299 (75% done) --- + [Epoch 226][10/66] loss=0.915952 avg=0.878995 VRAM=38.7GiB | 75.4% done | ETA(epoch): 1074s + [Epoch 226][20/66] loss=0.914884 avg=0.869380 VRAM=38.6GiB | 75.4% done | ETA(epoch): 882s + [Epoch 226][30/66] loss=0.791517 avg=0.861874 VRAM=38.7GiB | 75.5% done | ETA(epoch): 691s + [Epoch 226][40/66] loss=0.846646 avg=0.862516 VRAM=38.6GiB | 75.5% done | ETA(epoch): 499s + [Epoch 226][50/66] loss=0.775628 avg=0.858529 VRAM=38.7GiB | 75.6% done | ETA(epoch): 307s + [Epoch 226][60/66] loss=0.802256 avg=0.861901 VRAM=38.6GiB | 75.6% done | ETA(epoch): 115s + Train loss: 0.862275 (1267.9s) ETA: 1704min + Val loss: 0.848807 [t_0.0-0.2=1.0822 t_0.2-0.4=1.0073 t_0.4-0.6=0.8050 t_0.6-0.8=0.7341 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 227/299 (76% done) --- + [Epoch 227][10/66] loss=0.862175 avg=0.899143 VRAM=38.7GiB | 75.7% done | ETA(epoch): 1098s + [Epoch 227][20/66] loss=0.963315 avg=0.867513 VRAM=38.6GiB | 75.8% done | ETA(epoch): 893s + [Epoch 227][30/66] loss=0.810034 avg=0.867838 VRAM=38.7GiB | 75.8% done | ETA(epoch): 697s + [Epoch 227][40/66] loss=0.887836 avg=0.866887 VRAM=38.6GiB | 75.9% done | ETA(epoch): 502s + [Epoch 227][50/66] loss=0.731836 avg=0.862240 VRAM=38.7GiB | 75.9% done | ETA(epoch): 309s + [Epoch 227][60/66] loss=0.840907 avg=0.856186 VRAM=38.6GiB | 76.0% done | ETA(epoch): 116s + Train loss: 0.853320 (1271.8s) ETA: 1681min + Val loss: 0.838132 [t_0.0-0.2=1.0872 t_0.2-0.4=0.9881 t_0.4-0.6=0.8676 t_0.6-0.8=0.6882 t_0.8-1.0=0.6400] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 228/299 (76% done) --- + [Epoch 228][10/66] loss=0.846885 avg=0.872752 VRAM=38.7GiB | 76.1% done | ETA(epoch): 1074s + [Epoch 228][20/66] loss=0.865942 avg=0.869388 VRAM=38.6GiB | 76.1% done | ETA(epoch): 883s + [Epoch 228][30/66] loss=0.991987 avg=0.872107 VRAM=38.7GiB | 76.2% done | ETA(epoch): 691s + [Epoch 228][40/66] loss=0.883305 avg=0.866276 VRAM=38.6GiB | 76.2% done | ETA(epoch): 499s + [Epoch 228][50/66] loss=0.812380 avg=0.858902 VRAM=38.7GiB | 76.3% done | ETA(epoch): 307s + [Epoch 228][60/66] loss=0.858859 avg=0.855681 VRAM=38.6GiB | 76.3% done | ETA(epoch): 115s + Train loss: 0.857360 (1271.9s) ETA: 1657min + Val loss: 0.844468 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9900 t_0.4-0.6=0.8643 t_0.6-0.8=0.7068 t_0.8-1.0=0.6405] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 229/299 (76% done) --- + [Epoch 229][10/66] loss=0.848713 avg=0.852210 VRAM=38.7GiB | 76.4% done | ETA(epoch): 1074s + [Epoch 229][20/66] loss=0.955469 avg=0.859190 VRAM=38.6GiB | 76.4% done | ETA(epoch): 883s + [Epoch 229][30/66] loss=0.953135 avg=0.861338 VRAM=38.7GiB | 76.5% done | ETA(epoch): 692s + [Epoch 229][40/66] loss=0.814107 avg=0.863321 VRAM=38.6GiB | 76.5% done | ETA(epoch): 500s + [Epoch 229][50/66] loss=0.888357 avg=0.860039 VRAM=38.7GiB | 76.6% done | ETA(epoch): 307s + [Epoch 229][60/66] loss=0.836408 avg=0.852344 VRAM=38.6GiB | 76.6% done | ETA(epoch): 116s + Train loss: 0.849423 (1270.7s) ETA: 1634min + Val loss: 0.860125 [t_0.0-0.2=1.0851 t_0.2-0.4=1.0052 t_0.4-0.6=0.8304 t_0.6-0.8=0.7037 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 230/299 (77% done) --- + [Epoch 230][10/66] loss=0.932716 avg=0.877739 VRAM=38.7GiB | 76.7% done | ETA(epoch): 1077s + [Epoch 230][20/66] loss=0.872596 avg=0.873151 VRAM=38.6GiB | 76.8% done | ETA(epoch): 885s + [Epoch 230][30/66] loss=0.971305 avg=0.860515 VRAM=38.7GiB | 76.8% done | ETA(epoch): 692s + [Epoch 230][40/66] loss=0.954962 avg=0.866083 VRAM=38.6GiB | 76.9% done | ETA(epoch): 500s + [Epoch 230][50/66] loss=0.779808 avg=0.863074 VRAM=38.7GiB | 76.9% done | ETA(epoch): 309s + [Epoch 230][60/66] loss=0.875721 avg=0.861574 VRAM=38.6GiB | 77.0% done | ETA(epoch): 116s + Train loss: 0.860834 (1271.5s) ETA: 1610min + Val loss: 0.854880 [t_0.0-0.2=1.0924 t_0.2-0.4=0.9772 t_0.4-0.6=0.8525 t_0.6-0.8=0.6995 t_0.8-1.0=0.6454] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 231/299 (77% done) --- + [Epoch 231][10/66] loss=0.709717 avg=0.877166 VRAM=38.7GiB | 77.1% done | ETA(epoch): 1075s + [Epoch 231][20/66] loss=0.822698 avg=0.871149 VRAM=38.6GiB | 77.1% done | ETA(epoch): 884s + [Epoch 231][30/66] loss=0.883106 avg=0.865276 VRAM=38.7GiB | 77.2% done | ETA(epoch): 692s + [Epoch 231][40/66] loss=0.877157 avg=0.854933 VRAM=38.6GiB | 77.2% done | ETA(epoch): 501s + [Epoch 231][50/66] loss=0.787557 avg=0.859694 VRAM=38.7GiB | 77.3% done | ETA(epoch): 308s + [Epoch 231][60/66] loss=0.910379 avg=0.858099 VRAM=38.6GiB | 77.3% done | ETA(epoch): 115s + Train loss: 0.853756 (1269.6s) ETA: 1587min + Val loss: 0.848906 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0023 t_0.4-0.6=0.8439 t_0.6-0.8=0.7040 t_0.8-1.0=0.6302] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 232/299 (77% done) --- + [Epoch 232][10/66] loss=0.981920 avg=0.872498 VRAM=38.7GiB | 77.4% done | ETA(epoch): 1077s + [Epoch 232][20/66] loss=0.819133 avg=0.870437 VRAM=38.6GiB | 77.4% done | ETA(epoch): 884s + [Epoch 232][30/66] loss=0.693995 avg=0.857444 VRAM=38.7GiB | 77.5% done | ETA(epoch): 692s + [Epoch 232][40/66] loss=0.811225 avg=0.860048 VRAM=38.6GiB | 77.5% done | ETA(epoch): 500s + [Epoch 232][50/66] loss=0.877437 avg=0.857776 VRAM=38.7GiB | 77.6% done | ETA(epoch): 307s + [Epoch 232][60/66] loss=0.842139 avg=0.859325 VRAM=38.6GiB | 77.6% done | ETA(epoch): 115s + Train loss: 0.857073 (1268.1s) ETA: 1563min + Val loss: 0.876365 [t_0.0-0.2=1.0581 t_0.2-0.4=1.0116 t_0.4-0.6=0.8628 t_0.6-0.8=0.6989 t_0.8-1.0=0.6565] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 233/299 (78% done) --- + [Epoch 233][10/66] loss=0.871242 avg=0.887003 VRAM=38.7GiB | 77.7% done | ETA(epoch): 1075s + [Epoch 233][20/66] loss=0.892603 avg=0.869069 VRAM=38.6GiB | 77.8% done | ETA(epoch): 883s + [Epoch 233][30/66] loss=0.853064 avg=0.864630 VRAM=38.7GiB | 77.8% done | ETA(epoch): 692s + [Epoch 233][40/66] loss=0.847269 avg=0.861453 VRAM=38.6GiB | 77.9% done | ETA(epoch): 500s + [Epoch 233][50/66] loss=0.781307 avg=0.851486 VRAM=38.7GiB | 77.9% done | ETA(epoch): 307s + [Epoch 233][60/66] loss=0.880711 avg=0.852586 VRAM=38.6GiB | 78.0% done | ETA(epoch): 115s + Train loss: 0.847541 (1267.8s) ETA: 1540min + Val loss: 0.877529 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9640 t_0.4-0.6=0.8551 t_0.6-0.8=0.6874 t_0.8-1.0=0.6347] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 234/299 (78% done) --- + [Epoch 234][10/66] loss=0.752102 avg=0.912347 VRAM=38.7GiB | 78.1% done | ETA(epoch): 1074s + [Epoch 234][20/66] loss=0.814392 avg=0.886429 VRAM=38.6GiB | 78.1% done | ETA(epoch): 889s + [Epoch 234][30/66] loss=0.943109 avg=0.882204 VRAM=38.7GiB | 78.2% done | ETA(epoch): 694s + [Epoch 234][40/66] loss=0.868618 avg=0.872180 VRAM=38.6GiB | 78.2% done | ETA(epoch): 501s + [Epoch 234][50/66] loss=0.827305 avg=0.858890 VRAM=38.7GiB | 78.3% done | ETA(epoch): 308s + [Epoch 234][60/66] loss=0.753465 avg=0.853582 VRAM=38.6GiB | 78.3% done | ETA(epoch): 115s + Train loss: 0.853337 (1269.8s) ETA: 1516min + Val loss: 0.848464 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9886 t_0.4-0.6=0.8659 t_0.6-0.8=0.7013 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 235/299 (78% done) --- + [Epoch 235][10/66] loss=0.910781 avg=0.882386 VRAM=38.7GiB | 78.4% done | ETA(epoch): 1084s + [Epoch 235][20/66] loss=0.966199 avg=0.852650 VRAM=38.6GiB | 78.4% done | ETA(epoch): 887s + [Epoch 235][30/66] loss=0.912078 avg=0.848315 VRAM=38.7GiB | 78.5% done | ETA(epoch): 693s + [Epoch 235][40/66] loss=0.712090 avg=0.839851 VRAM=38.6GiB | 78.5% done | ETA(epoch): 500s + [Epoch 235][50/66] loss=0.873284 avg=0.839949 VRAM=38.7GiB | 78.6% done | ETA(epoch): 308s + [Epoch 235][60/66] loss=0.903980 avg=0.843254 VRAM=38.6GiB | 78.6% done | ETA(epoch): 115s + Train loss: 0.843725 (1270.9s) ETA: 1493min + Val loss: 0.869037 [t_0.0-0.2=1.0688 t_0.2-0.4=1.0055 t_0.4-0.6=0.8392 t_0.6-0.8=0.7173 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 236/299 (79% done) --- + [Epoch 236][10/66] loss=0.872937 avg=0.875758 VRAM=38.7GiB | 78.7% done | ETA(epoch): 1074s + [Epoch 236][20/66] loss=0.869493 avg=0.871318 VRAM=38.6GiB | 78.8% done | ETA(epoch): 883s + [Epoch 236][30/66] loss=0.799229 avg=0.876379 VRAM=38.7GiB | 78.8% done | ETA(epoch): 692s + [Epoch 236][40/66] loss=0.771659 avg=0.856064 VRAM=38.6GiB | 78.9% done | ETA(epoch): 499s + [Epoch 236][50/66] loss=0.907237 avg=0.856337 VRAM=38.7GiB | 78.9% done | ETA(epoch): 307s + [Epoch 236][60/66] loss=0.852203 avg=0.856495 VRAM=38.6GiB | 79.0% done | ETA(epoch): 116s + Train loss: 0.856412 (1270.3s) ETA: 1469min + Val loss: 0.863682 [t_0.0-0.2=1.0701 t_0.2-0.4=0.9870 t_0.4-0.6=0.8966 t_0.6-0.8=0.6899 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 237/299 (79% done) --- + [Epoch 237][10/66] loss=0.812350 avg=0.826718 VRAM=38.7GiB | 79.1% done | ETA(epoch): 1079s + [Epoch 237][20/66] loss=0.870323 avg=0.836763 VRAM=38.6GiB | 79.1% done | ETA(epoch): 885s + [Epoch 237][30/66] loss=0.985646 avg=0.844654 VRAM=38.7GiB | 79.2% done | ETA(epoch): 692s + [Epoch 237][40/66] loss=0.711072 avg=0.844246 VRAM=38.6GiB | 79.2% done | ETA(epoch): 500s + [Epoch 237][50/66] loss=0.776398 avg=0.840180 VRAM=38.7GiB | 79.3% done | ETA(epoch): 308s + [Epoch 237][60/66] loss=0.856292 avg=0.840884 VRAM=38.6GiB | 79.3% done | ETA(epoch): 115s + Train loss: 0.842491 (1269.0s) ETA: 1446min + Val loss: 0.851034 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9809 t_0.4-0.6=0.8680 t_0.6-0.8=0.7319 t_0.8-1.0=0.6498] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 238/299 (79% done) --- + [Epoch 238][10/66] loss=0.827730 avg=0.867073 VRAM=38.7GiB | 79.4% done | ETA(epoch): 1075s + [Epoch 238][20/66] loss=0.974059 avg=0.832639 VRAM=38.6GiB | 79.4% done | ETA(epoch): 883s + [Epoch 238][30/66] loss=0.746249 avg=0.834185 VRAM=38.7GiB | 79.5% done | ETA(epoch): 691s + [Epoch 238][40/66] loss=0.704199 avg=0.831796 VRAM=38.6GiB | 79.5% done | ETA(epoch): 499s + [Epoch 238][50/66] loss=0.898779 avg=0.839108 VRAM=38.7GiB | 79.6% done | ETA(epoch): 308s + [Epoch 238][60/66] loss=0.971033 avg=0.841139 VRAM=38.6GiB | 79.6% done | ETA(epoch): 116s + Train loss: 0.840394 (1270.3s) ETA: 1422min + Val loss: 0.865844 [t_0.0-0.2=1.0550 t_0.2-0.4=1.0107 t_0.4-0.6=0.8473 t_0.6-0.8=0.7171 t_0.8-1.0=0.6343] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 239/299 (80% done) --- + [Epoch 239][10/66] loss=0.895410 avg=0.845141 VRAM=38.7GiB | 79.7% done | ETA(epoch): 1076s + [Epoch 239][20/66] loss=0.751563 avg=0.839433 VRAM=38.6GiB | 79.8% done | ETA(epoch): 884s + [Epoch 239][30/66] loss=0.815522 avg=0.841615 VRAM=38.7GiB | 79.8% done | ETA(epoch): 691s + [Epoch 239][40/66] loss=0.887749 avg=0.852075 VRAM=38.6GiB | 79.9% done | ETA(epoch): 501s + [Epoch 239][50/66] loss=0.861113 avg=0.854157 VRAM=38.7GiB | 79.9% done | ETA(epoch): 308s + [Epoch 239][60/66] loss=0.891223 avg=0.853133 VRAM=38.6GiB | 80.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 15839 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 15839 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 15839 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 15839 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 15839 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 15839 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 15839 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 15839 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 15839 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 15839 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.854206 (2306.2s) ETA: 1403min + Val loss: 0.876596 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9969 t_0.4-0.6=0.8246 t_0.6-0.8=0.7089 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 240/299 (80% done) --- + [Epoch 240][10/66] loss=0.787140 avg=0.863296 VRAM=38.7GiB | 80.1% done | ETA(epoch): 1076s + [Epoch 240][20/66] loss=0.774624 avg=0.837799 VRAM=38.6GiB | 80.1% done | ETA(epoch): 884s + [Epoch 240][30/66] loss=0.703705 avg=0.838369 VRAM=38.7GiB | 80.2% done | ETA(epoch): 692s + [Epoch 240][40/66] loss=0.895897 avg=0.845769 VRAM=38.6GiB | 80.2% done | ETA(epoch): 502s + [Epoch 240][50/66] loss=0.842433 avg=0.848904 VRAM=38.7GiB | 80.3% done | ETA(epoch): 308s + [Epoch 240][60/66] loss=0.809377 avg=0.845360 VRAM=38.6GiB | 80.3% done | ETA(epoch): 116s + Train loss: 0.839855 (1271.7s) ETA: 1380min + Val loss: 0.878801 [t_0.0-0.2=1.0652 t_0.2-0.4=0.9927 t_0.4-0.6=0.8466 t_0.6-0.8=0.6924 t_0.8-1.0=0.6624] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 241/299 (80% done) --- + [Epoch 241][10/66] loss=0.912179 avg=0.857497 VRAM=38.7GiB | 80.4% done | ETA(epoch): 1075s + [Epoch 241][20/66] loss=0.767609 avg=0.870679 VRAM=38.6GiB | 80.4% done | ETA(epoch): 884s + [Epoch 241][30/66] loss=0.869425 avg=0.852299 VRAM=38.7GiB | 80.5% done | ETA(epoch): 694s + [Epoch 241][40/66] loss=0.741338 avg=0.843988 VRAM=38.6GiB | 80.5% done | ETA(epoch): 501s + [Epoch 241][50/66] loss=0.785167 avg=0.840787 VRAM=38.7GiB | 80.6% done | ETA(epoch): 308s + [Epoch 241][60/66] loss=0.846666 avg=0.842914 VRAM=38.6GiB | 80.6% done | ETA(epoch): 115s + Train loss: 0.843116 (1269.9s) ETA: 1356min + Val loss: 0.834031 [t_0.0-0.2=1.0598 t_0.2-0.4=0.9754 t_0.4-0.6=0.8433 t_0.6-0.8=0.7098 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 22.0/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 242/299 (81% done) --- + [Epoch 242][10/66] loss=0.870358 avg=0.846161 VRAM=38.7GiB | 80.7% done | ETA(epoch): 1075s + [Epoch 242][20/66] loss=0.945001 avg=0.877912 VRAM=38.6GiB | 80.8% done | ETA(epoch): 883s + [Epoch 242][30/66] loss=0.813352 avg=0.861710 VRAM=38.7GiB | 80.8% done | ETA(epoch): 692s + [Epoch 242][40/66] loss=0.998891 avg=0.864450 VRAM=38.6GiB | 80.9% done | ETA(epoch): 499s + [Epoch 242][50/66] loss=0.811622 avg=0.864600 VRAM=38.7GiB | 80.9% done | ETA(epoch): 307s + [Epoch 242][60/66] loss=0.838374 avg=0.863397 VRAM=38.6GiB | 81.0% done | ETA(epoch): 115s + Train loss: 0.863520 (1267.7s) ETA: 1333min + Val loss: 0.846071 [t_0.0-0.2=1.0724 t_0.2-0.4=1.0207 t_0.4-0.6=0.8473 t_0.6-0.8=0.7152 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 243/299 (81% done) --- + [Epoch 243][10/66] loss=0.920121 avg=0.849375 VRAM=38.7GiB | 81.1% done | ETA(epoch): 1076s + [Epoch 243][20/66] loss=0.905439 avg=0.834093 VRAM=38.6GiB | 81.1% done | ETA(epoch): 884s + [Epoch 243][30/66] loss=0.860592 avg=0.852482 VRAM=38.7GiB | 81.2% done | ETA(epoch): 692s + [Epoch 243][40/66] loss=0.708679 avg=0.855100 VRAM=38.6GiB | 81.2% done | ETA(epoch): 500s + [Epoch 243][50/66] loss=0.696346 avg=0.848628 VRAM=38.7GiB | 81.3% done | ETA(epoch): 308s + [Epoch 243][60/66] loss=0.720488 avg=0.842127 VRAM=38.6GiB | 81.3% done | ETA(epoch): 115s + Train loss: 0.846170 (1268.8s) ETA: 1309min + Val loss: 0.850091 [t_0.0-0.2=1.0850 t_0.2-0.4=0.9981 t_0.4-0.6=0.8732 t_0.6-0.8=0.7137 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 244/299 (81% done) --- + [Epoch 244][10/66] loss=0.729012 avg=0.852176 VRAM=38.7GiB | 81.4% done | ETA(epoch): 1076s + [Epoch 244][20/66] loss=0.843952 avg=0.878362 VRAM=38.6GiB | 81.4% done | ETA(epoch): 884s + [Epoch 244][30/66] loss=0.853844 avg=0.882030 VRAM=38.7GiB | 81.5% done | ETA(epoch): 692s + [Epoch 244][40/66] loss=0.838826 avg=0.873096 VRAM=38.6GiB | 81.5% done | ETA(epoch): 500s + [Epoch 244][50/66] loss=0.858385 avg=0.861785 VRAM=38.7GiB | 81.6% done | ETA(epoch): 308s + [Epoch 244][60/66] loss=0.868736 avg=0.863985 VRAM=38.6GiB | 81.6% done | ETA(epoch): 115s + Train loss: 0.863866 (1270.7s) ETA: 1286min + Val loss: 0.852134 [t_0.0-0.2=1.0810 t_0.2-0.4=1.0148 t_0.4-0.6=0.8295 t_0.6-0.8=0.7031 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 245/299 (82% done) --- + [Epoch 245][10/66] loss=0.937625 avg=0.846600 VRAM=38.7GiB | 81.7% done | ETA(epoch): 1074s + [Epoch 245][20/66] loss=0.778574 avg=0.866680 VRAM=38.6GiB | 81.8% done | ETA(epoch): 883s + [Epoch 245][30/66] loss=0.895449 avg=0.868537 VRAM=38.7GiB | 81.8% done | ETA(epoch): 691s + [Epoch 245][40/66] loss=0.913369 avg=0.858557 VRAM=38.6GiB | 81.9% done | ETA(epoch): 499s + [Epoch 245][50/66] loss=0.825185 avg=0.857480 VRAM=38.7GiB | 81.9% done | ETA(epoch): 307s + [Epoch 245][60/66] loss=0.889707 avg=0.853811 VRAM=38.6GiB | 82.0% done | ETA(epoch): 115s + Train loss: 0.855615 (1268.5s) ETA: 1262min + Val loss: 0.849604 [t_0.0-0.2=1.0487 t_0.2-0.4=1.0167 t_0.4-0.6=0.8699 t_0.6-0.8=0.6969 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0245 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 246/299 (82% done) --- + [Epoch 246][10/66] loss=0.862990 avg=0.885863 VRAM=38.7GiB | 82.1% done | ETA(epoch): 1075s + [Epoch 246][20/66] loss=0.824605 avg=0.886797 VRAM=38.6GiB | 82.1% done | ETA(epoch): 883s + [Epoch 246][30/66] loss=0.969709 avg=0.886833 VRAM=38.7GiB | 82.2% done | ETA(epoch): 692s + [Epoch 246][40/66] loss=0.942387 avg=0.872617 VRAM=38.6GiB | 82.2% done | ETA(epoch): 500s + [Epoch 246][50/66] loss=0.853779 avg=0.881030 VRAM=38.7GiB | 82.3% done | ETA(epoch): 308s + [Epoch 246][60/66] loss=0.862735 avg=0.878630 VRAM=38.6GiB | 82.3% done | ETA(epoch): 116s + Train loss: 0.871550 (1270.8s) ETA: 1239min + Val loss: 0.860198 [t_0.0-0.2=1.0532 t_0.2-0.4=1.0071 t_0.4-0.6=0.8610 t_0.6-0.8=0.7063 t_0.8-1.0=0.6442] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 22.0/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 247/299 (82% done) --- + [Epoch 247][10/66] loss=0.894396 avg=0.864008 VRAM=38.7GiB | 82.4% done | ETA(epoch): 1076s + [Epoch 247][20/66] loss=0.888493 avg=0.880235 VRAM=38.6GiB | 82.4% done | ETA(epoch): 884s + [Epoch 247][30/66] loss=0.782939 avg=0.876776 VRAM=38.7GiB | 82.5% done | ETA(epoch): 692s + [Epoch 247][40/66] loss=0.822438 avg=0.868351 VRAM=38.6GiB | 82.5% done | ETA(epoch): 500s + [Epoch 247][50/66] loss=0.967803 avg=0.868444 VRAM=38.7GiB | 82.6% done | ETA(epoch): 307s + [Epoch 247][60/66] loss=0.807544 avg=0.859738 VRAM=38.6GiB | 82.6% done | ETA(epoch): 115s + Train loss: 0.862784 (1268.6s) ETA: 1215min + Val loss: 0.833421 [t_0.0-0.2=1.0803 t_0.2-0.4=0.9697 t_0.4-0.6=0.8614 t_0.6-0.8=0.7071 t_0.8-1.0=0.6324] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 248/299 (83% done) --- + [Epoch 248][10/66] loss=0.874666 avg=0.802905 VRAM=38.7GiB | 82.7% done | ETA(epoch): 1075s + [Epoch 248][20/66] loss=0.765164 avg=0.815018 VRAM=38.6GiB | 82.8% done | ETA(epoch): 884s + [Epoch 248][30/66] loss=0.877772 avg=0.816061 VRAM=38.7GiB | 82.8% done | ETA(epoch): 692s + [Epoch 248][40/66] loss=0.890725 avg=0.824597 VRAM=38.6GiB | 82.9% done | ETA(epoch): 500s + [Epoch 248][50/66] loss=0.807145 avg=0.827525 VRAM=38.7GiB | 82.9% done | ETA(epoch): 308s + [Epoch 248][60/66] loss=0.873834 avg=0.832708 VRAM=38.6GiB | 83.0% done | ETA(epoch): 115s + Train loss: 0.838171 (1268.9s) ETA: 1192min + Val loss: 0.871448 [t_0.0-0.2=1.0638 t_0.2-0.4=0.9816 t_0.4-0.6=0.8443 t_0.6-0.8=0.7140 t_0.8-1.0=0.6440] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0248 + Deleted old checkpoint: checkpoint_epoch_0245 +[MEM @ epoch 248 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 249/299 (83% done) --- + [Epoch 249][10/66] loss=0.796068 avg=0.858691 VRAM=38.7GiB | 83.1% done | ETA(epoch): 1075s + [Epoch 249][20/66] loss=0.782815 avg=0.846773 VRAM=38.6GiB | 83.1% done | ETA(epoch): 883s + [Epoch 249][30/66] loss=0.847260 avg=0.848135 VRAM=38.7GiB | 83.2% done | ETA(epoch): 692s + [Epoch 249][40/66] loss=0.760924 avg=0.848605 VRAM=38.6GiB | 83.2% done | ETA(epoch): 500s + [Epoch 249][50/66] loss=0.904863 avg=0.840797 VRAM=38.7GiB | 83.3% done | ETA(epoch): 307s + [Epoch 249][60/66] loss=0.925921 avg=0.845903 VRAM=38.6GiB | 83.3% done | ETA(epoch): 115s + Train loss: 0.848154 (1268.0s) ETA: 1168min + Val loss: 0.841187 [t_0.0-0.2=1.0428 t_0.2-0.4=0.9799 t_0.4-0.6=0.8425 t_0.6-0.8=0.7372 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 250/299 (83% done) --- + [Epoch 250][10/66] loss=0.894084 avg=0.870129 VRAM=38.7GiB | 83.4% done | ETA(epoch): 1076s + [Epoch 250][20/66] loss=0.818025 avg=0.871779 VRAM=38.6GiB | 83.4% done | ETA(epoch): 884s + [Epoch 250][30/66] loss=0.886565 avg=0.872386 VRAM=38.7GiB | 83.5% done | ETA(epoch): 692s + [Epoch 250][40/66] loss=0.885947 avg=0.862387 VRAM=38.6GiB | 83.5% done | ETA(epoch): 500s + [Epoch 250][50/66] loss=0.765325 avg=0.858533 VRAM=38.7GiB | 83.6% done | ETA(epoch): 307s + [Epoch 250][60/66] loss=0.825293 avg=0.860608 VRAM=38.6GiB | 83.6% done | ETA(epoch): 115s + Train loss: 0.864750 (1268.1s) ETA: 1145min + Val loss: 0.873578 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0374 t_0.4-0.6=0.8140 t_0.6-0.8=0.7028 t_0.8-1.0=0.6560] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 251/299 (84% done) --- + [Epoch 251][10/66] loss=0.791023 avg=0.848134 VRAM=38.7GiB | 83.7% done | ETA(epoch): 1088s + [Epoch 251][20/66] loss=0.904123 avg=0.854735 VRAM=38.6GiB | 83.8% done | ETA(epoch): 888s + [Epoch 251][30/66] loss=1.000804 avg=0.851481 VRAM=38.7GiB | 83.8% done | ETA(epoch): 694s + [Epoch 251][40/66] loss=0.896419 avg=0.842626 VRAM=38.6GiB | 83.9% done | ETA(epoch): 501s + [Epoch 251][50/66] loss=0.874453 avg=0.849305 VRAM=38.7GiB | 83.9% done | ETA(epoch): 308s + [Epoch 251][60/66] loss=0.870213 avg=0.850910 VRAM=38.6GiB | 84.0% done | ETA(epoch): 116s + Train loss: 0.851227 (1270.5s) ETA: 1121min + Val loss: 0.834536 [t_0.0-0.2=1.0689 t_0.2-0.4=1.0130 t_0.4-0.6=0.8166 t_0.6-0.8=0.7057 t_0.8-1.0=0.6288] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 252/299 (84% done) --- + [Epoch 252][10/66] loss=0.900010 avg=0.843484 VRAM=38.7GiB | 84.1% done | ETA(epoch): 1076s + [Epoch 252][20/66] loss=0.962385 avg=0.845389 VRAM=38.6GiB | 84.1% done | ETA(epoch): 883s + [Epoch 252][30/66] loss=0.747069 avg=0.844062 VRAM=38.7GiB | 84.2% done | ETA(epoch): 691s + [Epoch 252][40/66] loss=0.773598 avg=0.842093 VRAM=38.6GiB | 84.2% done | ETA(epoch): 499s + [Epoch 252][50/66] loss=0.690155 avg=0.849126 VRAM=38.7GiB | 84.3% done | ETA(epoch): 307s + [Epoch 252][60/66] loss=0.964346 avg=0.856985 VRAM=38.6GiB | 84.3% done | ETA(epoch): 115s + Train loss: 0.856366 (1267.5s) ETA: 1098min + Val loss: 0.840749 [t_0.0-0.2=1.0656 t_0.2-0.4=1.0032 t_0.4-0.6=0.8322 t_0.6-0.8=0.7204 t_0.8-1.0=0.6333] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 23.4/188.5 GiB (12.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 253/299 (84% done) --- + [Epoch 253][10/66] loss=0.684275 avg=0.837892 VRAM=38.7GiB | 84.4% done | ETA(epoch): 1076s + [Epoch 253][20/66] loss=0.858235 avg=0.837075 VRAM=38.6GiB | 84.4% done | ETA(epoch): 883s + [Epoch 253][30/66] loss=0.903411 avg=0.858805 VRAM=38.7GiB | 84.5% done | ETA(epoch): 692s + [Epoch 253][40/66] loss=0.804553 avg=0.851308 VRAM=38.6GiB | 84.5% done | ETA(epoch): 500s + [Epoch 253][50/66] loss=0.903456 avg=0.854131 VRAM=38.7GiB | 84.6% done | ETA(epoch): 307s + [Epoch 253][60/66] loss=0.828712 avg=0.849585 VRAM=38.6GiB | 84.6% done | ETA(epoch): 115s + Train loss: 0.851042 (1268.1s) ETA: 1074min + Val loss: 0.848379 [t_0.0-0.2=1.0712 t_0.2-0.4=0.9921 t_0.4-0.6=0.8437 t_0.6-0.8=0.6995 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 254/299 (85% done) --- + [Epoch 254][10/66] loss=0.809953 avg=0.841180 VRAM=38.7GiB | 84.7% done | ETA(epoch): 1077s + [Epoch 254][20/66] loss=0.760078 avg=0.847949 VRAM=38.6GiB | 84.8% done | ETA(epoch): 884s + [Epoch 254][30/66] loss=0.817141 avg=0.848210 VRAM=38.7GiB | 84.8% done | ETA(epoch): 692s + [Epoch 254][40/66] loss=0.829085 avg=0.857483 VRAM=38.6GiB | 84.9% done | ETA(epoch): 500s + [Epoch 254][50/66] loss=0.957682 avg=0.859434 VRAM=38.7GiB | 84.9% done | ETA(epoch): 308s + [Epoch 254][60/66] loss=0.906189 avg=0.855983 VRAM=38.6GiB | 85.0% done | ETA(epoch): 115s + Train loss: 0.859093 (1269.0s) ETA: 1051min + Val loss: 0.880886 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0022 t_0.4-0.6=0.8568 t_0.6-0.8=0.7080 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 255/299 (85% done) --- + [Epoch 255][10/66] loss=0.843677 avg=0.853641 VRAM=38.7GiB | 85.1% done | ETA(epoch): 1074s + [Epoch 255][20/66] loss=0.774453 avg=0.840581 VRAM=38.6GiB | 85.1% done | ETA(epoch): 883s + [Epoch 255][30/66] loss=0.883752 avg=0.855174 VRAM=38.7GiB | 85.2% done | ETA(epoch): 691s + [Epoch 255][40/66] loss=0.928497 avg=0.858075 VRAM=38.6GiB | 85.2% done | ETA(epoch): 501s + [Epoch 255][50/66] loss=0.856970 avg=0.856864 VRAM=38.7GiB | 85.3% done | ETA(epoch): 308s + [Epoch 255][60/66] loss=0.909565 avg=0.855406 VRAM=38.6GiB | 85.3% done | ETA(epoch): 115s + Train loss: 0.856049 (1269.6s) ETA: 1027min + Val loss: 0.872086 [t_0.0-0.2=1.0628 t_0.2-0.4=0.9932 t_0.4-0.6=0.8627 t_0.6-0.8=0.7118 t_0.8-1.0=0.6341] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 23.2/188.5 GiB (12.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 256/299 (85% done) --- + [Epoch 256][10/66] loss=0.981425 avg=0.846604 VRAM=38.7GiB | 85.4% done | ETA(epoch): 1075s + [Epoch 256][20/66] loss=0.957981 avg=0.844380 VRAM=38.6GiB | 85.4% done | ETA(epoch): 883s + [Epoch 256][30/66] loss=0.954757 avg=0.844366 VRAM=38.7GiB | 85.5% done | ETA(epoch): 691s + [Epoch 256][40/66] loss=0.812134 avg=0.847590 VRAM=38.6GiB | 85.5% done | ETA(epoch): 499s + [Epoch 256][50/66] loss=0.942746 avg=0.848590 VRAM=38.7GiB | 85.6% done | ETA(epoch): 307s + [Epoch 256][60/66] loss=0.873548 avg=0.851288 VRAM=38.6GiB | 85.6% done | ETA(epoch): 115s + Train loss: 0.851761 (1268.2s) ETA: 1004min + Val loss: 0.859093 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9861 t_0.4-0.6=0.8485 t_0.6-0.8=0.7132 t_0.8-1.0=0.6479] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 23.4/188.5 GiB (12.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 257/299 (86% done) --- + [Epoch 257][10/66] loss=0.817333 avg=0.853536 VRAM=38.7GiB | 85.7% done | ETA(epoch): 1074s + [Epoch 257][20/66] loss=0.945451 avg=0.855219 VRAM=38.6GiB | 85.8% done | ETA(epoch): 883s + [Epoch 257][30/66] loss=0.820212 avg=0.845665 VRAM=38.7GiB | 85.8% done | ETA(epoch): 691s + [Epoch 257][40/66] loss=0.815575 avg=0.841267 VRAM=38.6GiB | 85.9% done | ETA(epoch): 499s + [Epoch 257][50/66] loss=0.927717 avg=0.846238 VRAM=38.7GiB | 85.9% done | ETA(epoch): 307s + [Epoch 257][60/66] loss=0.884299 avg=0.855853 VRAM=38.6GiB | 86.0% done | ETA(epoch): 115s + Train loss: 0.852596 (1267.0s) ETA: 980min + Val loss: 0.840893 [t_0.0-0.2=1.0590 t_0.2-0.4=1.0291 t_0.4-0.6=0.8365 t_0.6-0.8=0.7074 t_0.8-1.0=0.6237] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 22.9/188.5 GiB (12.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 258/299 (86% done) --- + [Epoch 258][10/66] loss=0.952319 avg=0.907028 VRAM=38.7GiB | 86.1% done | ETA(epoch): 1076s + [Epoch 258][20/66] loss=0.724133 avg=0.872093 VRAM=38.6GiB | 86.1% done | ETA(epoch): 884s + [Epoch 258][30/66] loss=0.763557 avg=0.856502 VRAM=38.7GiB | 86.2% done | ETA(epoch): 691s + [Epoch 258][40/66] loss=0.857859 avg=0.856379 VRAM=38.6GiB | 86.2% done | ETA(epoch): 499s + [Epoch 258][50/66] loss=0.939018 avg=0.860832 VRAM=38.7GiB | 86.3% done | ETA(epoch): 307s + [Epoch 258][60/66] loss=0.865154 avg=0.859913 VRAM=38.6GiB | 86.3% done | ETA(epoch): 115s + Train loss: 0.855585 (1268.2s) ETA: 957min + Val loss: 0.901042 [t_0.0-0.2=1.1005 t_0.2-0.4=0.9905 t_0.4-0.6=0.8655 t_0.6-0.8=0.6945 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 23.0/188.5 GiB (12.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 259/299 (86% done) --- + [Epoch 259][10/66] loss=0.866505 avg=0.897598 VRAM=38.7GiB | 86.4% done | ETA(epoch): 1090s + [Epoch 259][20/66] loss=0.831711 avg=0.864240 VRAM=38.6GiB | 86.4% done | ETA(epoch): 889s + [Epoch 259][30/66] loss=0.899810 avg=0.868243 VRAM=38.7GiB | 86.5% done | ETA(epoch): 695s + [Epoch 259][40/66] loss=0.809634 avg=0.864590 VRAM=38.6GiB | 86.5% done | ETA(epoch): 501s + [Epoch 259][50/66] loss=0.899958 avg=0.864420 VRAM=38.7GiB | 86.6% done | ETA(epoch): 308s + [Epoch 259][60/66] loss=1.011132 avg=0.864439 VRAM=38.6GiB | 86.6% done | ETA(epoch): 116s + Train loss: 0.859838 (1272.5s) ETA: 934min + Val loss: 0.864796 [t_0.0-0.2=1.0496 t_0.2-0.4=0.9899 t_0.4-0.6=0.8695 t_0.6-0.8=0.7302 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 23.5/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 260/299 (87% done) --- + [Epoch 260][10/66] loss=0.921514 avg=0.841827 VRAM=38.7GiB | 86.7% done | ETA(epoch): 1075s + [Epoch 260][20/66] loss=0.805934 avg=0.853031 VRAM=38.6GiB | 86.8% done | ETA(epoch): 884s + [Epoch 260][30/66] loss=0.911751 avg=0.861540 VRAM=38.7GiB | 86.8% done | ETA(epoch): 692s + [Epoch 260][40/66] loss=0.732566 avg=0.864653 VRAM=38.6GiB | 86.9% done | ETA(epoch): 500s + [Epoch 260][50/66] loss=0.859111 avg=0.854990 VRAM=38.7GiB | 86.9% done | ETA(epoch): 308s + [Epoch 260][60/66] loss=0.726051 avg=0.854503 VRAM=38.6GiB | 87.0% done | ETA(epoch): 115s + Train loss: 0.855151 (1268.9s) ETA: 910min + Val loss: 0.821787 [t_0.0-0.2=1.0636 t_0.2-0.4=0.9857 t_0.4-0.6=0.8218 t_0.6-0.8=0.7205 t_0.8-1.0=0.6488] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 261/299 (87% done) --- + [Epoch 261][10/66] loss=0.879369 avg=0.852816 VRAM=38.7GiB | 87.1% done | ETA(epoch): 1076s + [Epoch 261][20/66] loss=0.840933 avg=0.839104 VRAM=38.6GiB | 87.1% done | ETA(epoch): 884s + [Epoch 261][30/66] loss=0.843040 avg=0.853692 VRAM=38.7GiB | 87.2% done | ETA(epoch): 691s + [Epoch 261][40/66] loss=0.914123 avg=0.859230 VRAM=38.6GiB | 87.2% done | ETA(epoch): 499s + [Epoch 261][50/66] loss=0.951717 avg=0.865337 VRAM=38.7GiB | 87.3% done | ETA(epoch): 309s + [Epoch 261][60/66] loss=0.836306 avg=0.863504 VRAM=38.6GiB | 87.3% done | ETA(epoch): 116s + Train loss: 0.860400 (1271.3s) ETA: 887min + Val loss: 0.831433 [t_0.0-0.2=1.0691 t_0.2-0.4=1.0386 t_0.4-0.6=0.8516 t_0.6-0.8=0.6955 t_0.8-1.0=0.6478] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 262/299 (87% done) --- + [Epoch 262][10/66] loss=0.840978 avg=0.870812 VRAM=38.7GiB | 87.4% done | ETA(epoch): 1075s + [Epoch 262][20/66] loss=0.837647 avg=0.873915 VRAM=38.6GiB | 87.4% done | ETA(epoch): 884s + [Epoch 262][30/66] loss=0.714126 avg=0.845724 VRAM=38.7GiB | 87.5% done | ETA(epoch): 692s + [Epoch 262][40/66] loss=0.911270 avg=0.855105 VRAM=38.6GiB | 87.5% done | ETA(epoch): 500s + [Epoch 262][50/66] loss=1.000942 avg=0.860010 VRAM=38.7GiB | 87.6% done | ETA(epoch): 307s + [Epoch 262][60/66] loss=0.862904 avg=0.859640 VRAM=38.6GiB | 87.6% done | ETA(epoch): 115s + Train loss: 0.859443 (1268.2s) ETA: 863min + Val loss: 0.877688 [t_0.0-0.2=1.0794 t_0.2-0.4=1.0241 t_0.4-0.6=0.8563 t_0.6-0.8=0.6974 t_0.8-1.0=0.6585] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 263/299 (88% done) --- + [Epoch 263][10/66] loss=0.851733 avg=0.843263 VRAM=38.7GiB | 87.7% done | ETA(epoch): 1074s + [Epoch 263][20/66] loss=0.792869 avg=0.848642 VRAM=38.6GiB | 87.8% done | ETA(epoch): 882s + [Epoch 263][30/66] loss=0.822719 avg=0.848815 VRAM=38.7GiB | 87.8% done | ETA(epoch): 691s + [Epoch 263][40/66] loss=0.834471 avg=0.853263 VRAM=38.6GiB | 87.9% done | ETA(epoch): 501s + [Epoch 263][50/66] loss=0.809293 avg=0.840416 VRAM=38.7GiB | 87.9% done | ETA(epoch): 308s + [Epoch 263][60/66] loss=0.774475 avg=0.840743 VRAM=38.6GiB | 88.0% done | ETA(epoch): 115s + Train loss: 0.845666 (1269.4s) ETA: 840min + Val loss: 0.889425 [t_0.0-0.2=1.0844 t_0.2-0.4=1.0045 t_0.4-0.6=0.8550 t_0.6-0.8=0.7345 t_0.8-1.0=0.6465] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 264/299 (88% done) --- + [Epoch 264][10/66] loss=0.871289 avg=0.888291 VRAM=38.7GiB | 88.1% done | ETA(epoch): 1075s + [Epoch 264][20/66] loss=0.746918 avg=0.853673 VRAM=38.6GiB | 88.1% done | ETA(epoch): 883s + [Epoch 264][30/66] loss=0.774383 avg=0.846729 VRAM=38.7GiB | 88.2% done | ETA(epoch): 694s + [Epoch 264][40/66] loss=0.864141 avg=0.849112 VRAM=38.6GiB | 88.2% done | ETA(epoch): 501s + [Epoch 264][50/66] loss=0.803138 avg=0.844880 VRAM=38.7GiB | 88.3% done | ETA(epoch): 308s + [Epoch 264][60/66] loss=0.867950 avg=0.843605 VRAM=38.6GiB | 88.3% done | ETA(epoch): 115s + Train loss: 0.843207 (1270.1s) ETA: 816min + Val loss: 0.880174 [t_0.0-0.2=1.0645 t_0.2-0.4=0.9919 t_0.4-0.6=0.8869 t_0.6-0.8=0.7173 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 265/299 (88% done) --- + [Epoch 265][10/66] loss=0.877253 avg=0.802285 VRAM=38.7GiB | 88.4% done | ETA(epoch): 1078s + [Epoch 265][20/66] loss=0.781823 avg=0.846104 VRAM=38.6GiB | 88.4% done | ETA(epoch): 885s + [Epoch 265][30/66] loss=0.795703 avg=0.830161 VRAM=38.7GiB | 88.5% done | ETA(epoch): 692s + [Epoch 265][40/66] loss=0.865242 avg=0.837912 VRAM=38.6GiB | 88.5% done | ETA(epoch): 500s + [Epoch 265][50/66] loss=0.825094 avg=0.842559 VRAM=38.7GiB | 88.6% done | ETA(epoch): 308s + [Epoch 265][60/66] loss=0.903953 avg=0.847875 VRAM=38.6GiB | 88.6% done | ETA(epoch): 115s + Train loss: 0.846776 (1268.6s) ETA: 793min + Val loss: 0.816033 [t_0.0-0.2=1.0497 t_0.2-0.4=0.9750 t_0.4-0.6=0.8451 t_0.6-0.8=0.7213 t_0.8-1.0=0.6158] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 24.1/188.5 GiB (12.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 266/299 (89% done) --- + [Epoch 266][10/66] loss=0.858369 avg=0.827048 VRAM=38.7GiB | 88.7% done | ETA(epoch): 1074s + [Epoch 266][20/66] loss=1.025254 avg=0.840994 VRAM=38.6GiB | 88.8% done | ETA(epoch): 884s + [Epoch 266][30/66] loss=0.831782 avg=0.840449 VRAM=38.7GiB | 88.8% done | ETA(epoch): 692s + [Epoch 266][40/66] loss=0.894611 avg=0.847680 VRAM=38.6GiB | 88.9% done | ETA(epoch): 500s + [Epoch 266][50/66] loss=0.777001 avg=0.843925 VRAM=38.7GiB | 88.9% done | ETA(epoch): 307s + [Epoch 266][60/66] loss=0.863950 avg=0.847493 VRAM=38.6GiB | 89.0% done | ETA(epoch): 115s + Train loss: 0.851658 (1268.1s) ETA: 770min + Val loss: 0.820475 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9836 t_0.4-0.6=0.8123 t_0.6-0.8=0.7084 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 267/299 (89% done) --- + [Epoch 267][10/66] loss=0.880332 avg=0.872434 VRAM=38.7GiB | 89.1% done | ETA(epoch): 1075s + [Epoch 267][20/66] loss=0.796683 avg=0.857477 VRAM=38.6GiB | 89.1% done | ETA(epoch): 883s + [Epoch 267][30/66] loss=0.889493 avg=0.863686 VRAM=38.7GiB | 89.2% done | ETA(epoch): 691s + [Epoch 267][40/66] loss=0.714275 avg=0.858819 VRAM=38.6GiB | 89.2% done | ETA(epoch): 499s + [Epoch 267][50/66] loss=0.892087 avg=0.856967 VRAM=38.7GiB | 89.3% done | ETA(epoch): 307s + [Epoch 267][60/66] loss=0.893090 avg=0.860385 VRAM=38.6GiB | 89.3% done | ETA(epoch): 115s + Train loss: 0.861463 (1268.2s) ETA: 746min + Val loss: 0.887458 [t_0.0-0.2=1.0915 t_0.2-0.4=0.9805 t_0.4-0.6=0.8641 t_0.6-0.8=0.6945 t_0.8-1.0=0.6393] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 268/299 (89% done) --- + [Epoch 268][10/66] loss=0.830330 avg=0.881259 VRAM=38.7GiB | 89.4% done | ETA(epoch): 1075s + [Epoch 268][20/66] loss=0.912958 avg=0.851650 VRAM=38.6GiB | 89.4% done | ETA(epoch): 883s + [Epoch 268][30/66] loss=0.828455 avg=0.856869 VRAM=38.7GiB | 89.5% done | ETA(epoch): 692s + [Epoch 268][40/66] loss=0.870790 avg=0.865022 VRAM=38.6GiB | 89.5% done | ETA(epoch): 499s + [Epoch 268][50/66] loss=0.940972 avg=0.864713 VRAM=38.7GiB | 89.6% done | ETA(epoch): 307s + [Epoch 268][60/66] loss=0.760019 avg=0.863447 VRAM=38.6GiB | 89.6% done | ETA(epoch): 115s + Train loss: 0.864374 (1267.7s) ETA: 723min + Val loss: 0.874273 [t_0.0-0.2=1.0617 t_0.2-0.4=1.0062 t_0.4-0.6=0.8645 t_0.6-0.8=0.7095 t_0.8-1.0=0.6155] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 269/299 (90% done) --- + [Epoch 269][10/66] loss=0.910637 avg=0.897276 VRAM=38.7GiB | 89.7% done | ETA(epoch): 1075s + [Epoch 269][20/66] loss=0.883642 avg=0.875626 VRAM=38.6GiB | 89.8% done | ETA(epoch): 883s + [Epoch 269][30/66] loss=0.839080 avg=0.859921 VRAM=38.7GiB | 89.8% done | ETA(epoch): 691s + [Epoch 269][40/66] loss=0.873243 avg=0.859926 VRAM=38.6GiB | 89.9% done | ETA(epoch): 499s + [Epoch 269][50/66] loss=0.890825 avg=0.857572 VRAM=38.7GiB | 89.9% done | ETA(epoch): 308s + [Epoch 269][60/66] loss=0.762237 avg=0.852493 VRAM=38.6GiB | 90.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 17819 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 17819 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 17819 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 17819 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 17819 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 17819 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 17819 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 17819 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 17819 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 17819 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.852377 (2311.1s) ETA: 701min + Val loss: 0.835930 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9765 t_0.4-0.6=0.8487 t_0.6-0.8=0.7105 t_0.8-1.0=0.6254] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 270/299 (90% done) --- + [Epoch 270][10/66] loss=0.803457 avg=0.856387 VRAM=38.7GiB | 90.1% done | ETA(epoch): 1076s + [Epoch 270][20/66] loss=0.794087 avg=0.845150 VRAM=38.6GiB | 90.1% done | ETA(epoch): 884s + [Epoch 270][30/66] loss=0.931698 avg=0.861029 VRAM=38.7GiB | 90.2% done | ETA(epoch): 692s + [Epoch 270][40/66] loss=0.875246 avg=0.856395 VRAM=38.6GiB | 90.2% done | ETA(epoch): 500s + [Epoch 270][50/66] loss=0.749314 avg=0.847630 VRAM=38.7GiB | 90.3% done | ETA(epoch): 308s + [Epoch 270][60/66] loss=0.879891 avg=0.852052 VRAM=38.6GiB | 90.3% done | ETA(epoch): 115s + Train loss: 0.848097 (1268.4s) ETA: 678min + Val loss: 0.851113 [t_0.0-0.2=1.0584 t_0.2-0.4=1.0264 t_0.4-0.6=0.8691 t_0.6-0.8=0.7360 t_0.8-1.0=0.6244] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 271/299 (90% done) --- + [Epoch 271][10/66] loss=0.800283 avg=0.870540 VRAM=38.7GiB | 90.4% done | ETA(epoch): 1075s + [Epoch 271][20/66] loss=0.882211 avg=0.861349 VRAM=38.6GiB | 90.4% done | ETA(epoch): 884s + [Epoch 271][30/66] loss=0.759678 avg=0.854100 VRAM=38.7GiB | 90.5% done | ETA(epoch): 692s + [Epoch 271][40/66] loss=0.869031 avg=0.854120 VRAM=38.6GiB | 90.5% done | ETA(epoch): 500s + [Epoch 271][50/66] loss=0.826995 avg=0.854474 VRAM=38.7GiB | 90.6% done | ETA(epoch): 307s + [Epoch 271][60/66] loss=0.847703 avg=0.856223 VRAM=38.6GiB | 90.6% done | ETA(epoch): 115s + Train loss: 0.856576 (1268.7s) ETA: 654min + Val loss: 0.841879 [t_0.0-0.2=1.0618 t_0.2-0.4=1.0082 t_0.4-0.6=0.8176 t_0.6-0.8=0.7336 t_0.8-1.0=0.6433] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 272/299 (91% done) --- + [Epoch 272][10/66] loss=0.720875 avg=0.831022 VRAM=38.7GiB | 90.7% done | ETA(epoch): 1075s + [Epoch 272][20/66] loss=0.834659 avg=0.853731 VRAM=38.6GiB | 90.8% done | ETA(epoch): 883s + [Epoch 272][30/66] loss=0.968291 avg=0.866415 VRAM=38.7GiB | 90.8% done | ETA(epoch): 694s + [Epoch 272][40/66] loss=0.969220 avg=0.872872 VRAM=38.6GiB | 90.9% done | ETA(epoch): 501s + [Epoch 272][50/66] loss=0.824965 avg=0.869128 VRAM=38.7GiB | 90.9% done | ETA(epoch): 308s + [Epoch 272][60/66] loss=0.735578 avg=0.866502 VRAM=38.6GiB | 91.0% done | ETA(epoch): 115s + Train loss: 0.873842 (1270.2s) ETA: 631min + Val loss: 0.810344 [t_0.0-0.2=1.0434 t_0.2-0.4=0.9955 t_0.4-0.6=0.8576 t_0.6-0.8=0.6798 t_0.8-1.0=0.6438] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 273/299 (91% done) --- + [Epoch 273][10/66] loss=0.898277 avg=0.863984 VRAM=38.7GiB | 91.1% done | ETA(epoch): 1075s + [Epoch 273][20/66] loss=0.952804 avg=0.859807 VRAM=38.6GiB | 91.1% done | ETA(epoch): 886s + [Epoch 273][30/66] loss=0.837125 avg=0.851133 VRAM=38.7GiB | 91.2% done | ETA(epoch): 693s + [Epoch 273][40/66] loss=0.811442 avg=0.853741 VRAM=38.6GiB | 91.2% done | ETA(epoch): 500s + [Epoch 273][50/66] loss=0.967587 avg=0.855065 VRAM=38.7GiB | 91.3% done | ETA(epoch): 308s + [Epoch 273][60/66] loss=0.807441 avg=0.856407 VRAM=38.6GiB | 91.3% done | ETA(epoch): 115s + Train loss: 0.856018 (1269.0s) ETA: 608min + Val loss: 0.873540 [t_0.0-0.2=1.0699 t_0.2-0.4=0.9870 t_0.4-0.6=0.8664 t_0.6-0.8=0.7170 t_0.8-1.0=0.6369] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 23.2/188.5 GiB (12.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 274/299 (91% done) --- + [Epoch 274][10/66] loss=0.910890 avg=0.829592 VRAM=38.7GiB | 91.4% done | ETA(epoch): 1074s + [Epoch 274][20/66] loss=0.783227 avg=0.839866 VRAM=38.6GiB | 91.4% done | ETA(epoch): 883s + [Epoch 274][30/66] loss=0.838346 avg=0.835261 VRAM=38.7GiB | 91.5% done | ETA(epoch): 691s + [Epoch 274][40/66] loss=0.941285 avg=0.848166 VRAM=38.6GiB | 91.5% done | ETA(epoch): 499s + [Epoch 274][50/66] loss=0.728190 avg=0.844830 VRAM=38.7GiB | 91.6% done | ETA(epoch): 307s + [Epoch 274][60/66] loss=0.808683 avg=0.840785 VRAM=38.6GiB | 91.6% done | ETA(epoch): 115s + Train loss: 0.840428 (1267.6s) ETA: 584min + Val loss: 0.868347 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9929 t_0.4-0.6=0.8653 t_0.6-0.8=0.7126 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 275/299 (92% done) --- + [Epoch 275][10/66] loss=0.872273 avg=0.848095 VRAM=38.7GiB | 91.7% done | ETA(epoch): 1074s + [Epoch 275][20/66] loss=0.927693 avg=0.849619 VRAM=38.6GiB | 91.8% done | ETA(epoch): 883s + [Epoch 275][30/66] loss=0.870334 avg=0.845172 VRAM=38.7GiB | 91.8% done | ETA(epoch): 691s + [Epoch 275][40/66] loss=0.881065 avg=0.851035 VRAM=38.6GiB | 91.9% done | ETA(epoch): 499s + [Epoch 275][50/66] loss=0.792246 avg=0.850652 VRAM=38.7GiB | 91.9% done | ETA(epoch): 307s + [Epoch 275][60/66] loss=0.939721 avg=0.849584 VRAM=38.6GiB | 92.0% done | ETA(epoch): 115s + Train loss: 0.850418 (1268.4s) ETA: 561min + Val loss: 0.847669 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9639 t_0.4-0.6=0.8770 t_0.6-0.8=0.7109 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 276/299 (92% done) --- + [Epoch 276][10/66] loss=0.860525 avg=0.853552 VRAM=38.7GiB | 92.1% done | ETA(epoch): 1074s + [Epoch 276][20/66] loss=0.854595 avg=0.847901 VRAM=38.6GiB | 92.1% done | ETA(epoch): 883s + [Epoch 276][30/66] loss=0.849901 avg=0.850350 VRAM=38.7GiB | 92.2% done | ETA(epoch): 692s + [Epoch 276][40/66] loss=0.797401 avg=0.845602 VRAM=38.6GiB | 92.2% done | ETA(epoch): 500s + [Epoch 276][50/66] loss=0.807652 avg=0.842622 VRAM=38.7GiB | 92.3% done | ETA(epoch): 308s + [Epoch 276][60/66] loss=0.805451 avg=0.837284 VRAM=38.6GiB | 92.3% done | ETA(epoch): 115s + Train loss: 0.840487 (1268.6s) ETA: 537min + Val loss: 0.862763 [t_0.0-0.2=1.0740 t_0.2-0.4=1.0149 t_0.4-0.6=0.8335 t_0.6-0.8=0.7264 t_0.8-1.0=0.6405] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 277/299 (92% done) --- + [Epoch 277][10/66] loss=0.864155 avg=0.829124 VRAM=38.7GiB | 92.4% done | ETA(epoch): 1074s + [Epoch 277][20/66] loss=0.995462 avg=0.851279 VRAM=38.6GiB | 92.4% done | ETA(epoch): 883s + [Epoch 277][30/66] loss=0.892569 avg=0.850602 VRAM=38.7GiB | 92.5% done | ETA(epoch): 691s + [Epoch 277][40/66] loss=0.856896 avg=0.858217 VRAM=38.6GiB | 92.5% done | ETA(epoch): 499s + [Epoch 277][50/66] loss=1.018945 avg=0.858600 VRAM=38.7GiB | 92.6% done | ETA(epoch): 308s + [Epoch 277][60/66] loss=0.942439 avg=0.857772 VRAM=38.6GiB | 92.6% done | ETA(epoch): 116s + Train loss: 0.858138 (1271.4s) ETA: 514min + Val loss: 0.878104 [t_0.0-0.2=1.0728 t_0.2-0.4=0.9803 t_0.4-0.6=0.8301 t_0.6-0.8=0.7315 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 278/299 (93% done) --- + [Epoch 278][10/66] loss=0.791861 avg=0.865217 VRAM=38.7GiB | 92.7% done | ETA(epoch): 1075s + [Epoch 278][20/66] loss=0.846583 avg=0.868748 VRAM=38.6GiB | 92.8% done | ETA(epoch): 883s + [Epoch 278][30/66] loss=0.808265 avg=0.850146 VRAM=38.7GiB | 92.8% done | ETA(epoch): 691s + [Epoch 278][40/66] loss=0.919343 avg=0.849991 VRAM=38.6GiB | 92.9% done | ETA(epoch): 499s + [Epoch 278][50/66] loss=0.814867 avg=0.855439 VRAM=38.7GiB | 92.9% done | ETA(epoch): 308s + [Epoch 278][60/66] loss=0.818293 avg=0.852454 VRAM=38.6GiB | 93.0% done | ETA(epoch): 115s + Train loss: 0.853288 (1269.7s) ETA: 491min + Val loss: 0.864988 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9882 t_0.4-0.6=0.8290 t_0.6-0.8=0.7296 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 279/299 (93% done) --- + [Epoch 279][10/66] loss=0.792819 avg=0.842444 VRAM=38.7GiB | 93.1% done | ETA(epoch): 1077s + [Epoch 279][20/66] loss=0.758525 avg=0.834814 VRAM=38.6GiB | 93.1% done | ETA(epoch): 884s + [Epoch 279][30/66] loss=0.758990 avg=0.837574 VRAM=38.7GiB | 93.2% done | ETA(epoch): 693s + [Epoch 279][40/66] loss=0.973292 avg=0.849792 VRAM=38.6GiB | 93.2% done | ETA(epoch): 503s + [Epoch 279][50/66] loss=0.803902 avg=0.849065 VRAM=38.7GiB | 93.3% done | ETA(epoch): 309s + [Epoch 279][60/66] loss=0.722092 avg=0.846982 VRAM=38.6GiB | 93.3% done | ETA(epoch): 116s + Train loss: 0.852049 (1273.7s) ETA: 467min + Val loss: 0.828790 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0224 t_0.4-0.6=0.8580 t_0.6-0.8=0.6830 t_0.8-1.0=0.6366] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 280/299 (93% done) --- + [Epoch 280][10/66] loss=0.951036 avg=0.855300 VRAM=38.7GiB | 93.4% done | ETA(epoch): 1074s + [Epoch 280][20/66] loss=0.775690 avg=0.862887 VRAM=38.6GiB | 93.4% done | ETA(epoch): 884s + [Epoch 280][30/66] loss=0.756618 avg=0.849277 VRAM=38.7GiB | 93.5% done | ETA(epoch): 692s + [Epoch 280][40/66] loss=0.696419 avg=0.843371 VRAM=38.6GiB | 93.5% done | ETA(epoch): 500s + [Epoch 280][50/66] loss=0.933570 avg=0.846161 VRAM=38.7GiB | 93.6% done | ETA(epoch): 308s + [Epoch 280][60/66] loss=0.878570 avg=0.847373 VRAM=38.6GiB | 93.6% done | ETA(epoch): 115s + Train loss: 0.852131 (1269.3s) ETA: 444min + Val loss: 0.869870 [t_0.0-0.2=1.0837 t_0.2-0.4=1.0197 t_0.4-0.6=0.8159 t_0.6-0.8=0.6850 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 281/299 (94% done) --- + [Epoch 281][10/66] loss=0.844327 avg=0.848540 VRAM=38.7GiB | 93.7% done | ETA(epoch): 1076s + [Epoch 281][20/66] loss=0.893677 avg=0.860447 VRAM=38.6GiB | 93.8% done | ETA(epoch): 883s + [Epoch 281][30/66] loss=0.834483 avg=0.853282 VRAM=38.7GiB | 93.8% done | ETA(epoch): 692s + [Epoch 281][40/66] loss=0.756821 avg=0.854126 VRAM=38.6GiB | 93.9% done | ETA(epoch): 500s + [Epoch 281][50/66] loss=0.963797 avg=0.855446 VRAM=38.7GiB | 93.9% done | ETA(epoch): 308s + [Epoch 281][60/66] loss=1.001598 avg=0.860530 VRAM=38.6GiB | 94.0% done | ETA(epoch): 115s + Train loss: 0.857672 (1269.1s) ETA: 420min + Val loss: 0.799002 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9908 t_0.4-0.6=0.8391 t_0.6-0.8=0.6995 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0281 (BEST) + Deleted old checkpoint: checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 282/299 (94% done) --- + [Epoch 282][10/66] loss=0.775729 avg=0.815379 VRAM=38.7GiB | 94.1% done | ETA(epoch): 1077s + [Epoch 282][20/66] loss=0.933401 avg=0.868789 VRAM=38.6GiB | 94.1% done | ETA(epoch): 884s + [Epoch 282][30/66] loss=1.029049 avg=0.877673 VRAM=38.7GiB | 94.2% done | ETA(epoch): 692s + [Epoch 282][40/66] loss=0.856307 avg=0.861603 VRAM=38.6GiB | 94.2% done | ETA(epoch): 500s + [Epoch 282][50/66] loss=0.834165 avg=0.862801 VRAM=38.7GiB | 94.3% done | ETA(epoch): 308s + [Epoch 282][60/66] loss=0.914447 avg=0.854024 VRAM=38.6GiB | 94.3% done | ETA(epoch): 115s + Train loss: 0.853460 (1268.7s) ETA: 397min + Val loss: 0.813957 [t_0.0-0.2=1.0594 t_0.2-0.4=1.0022 t_0.4-0.6=0.8432 t_0.6-0.8=0.7028 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 283/299 (94% done) --- + [Epoch 283][10/66] loss=0.900681 avg=0.855126 VRAM=38.7GiB | 94.4% done | ETA(epoch): 1075s + [Epoch 283][20/66] loss=0.892057 avg=0.866753 VRAM=38.6GiB | 94.4% done | ETA(epoch): 884s + [Epoch 283][30/66] loss=0.789576 avg=0.858098 VRAM=38.7GiB | 94.5% done | ETA(epoch): 691s + [Epoch 283][40/66] loss=0.768705 avg=0.856614 VRAM=38.6GiB | 94.5% done | ETA(epoch): 500s + [Epoch 283][50/66] loss=0.924975 avg=0.856054 VRAM=38.7GiB | 94.6% done | ETA(epoch): 308s + [Epoch 283][60/66] loss=0.791348 avg=0.854062 VRAM=38.6GiB | 94.6% done | ETA(epoch): 115s + Train loss: 0.851124 (1269.0s) ETA: 374min + Val loss: 0.843419 [t_0.0-0.2=1.0766 t_0.2-0.4=1.0010 t_0.4-0.6=0.8398 t_0.6-0.8=0.6939 t_0.8-1.0=0.6264] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 284/299 (95% done) --- + [Epoch 284][10/66] loss=0.939206 avg=0.870777 VRAM=38.7GiB | 94.7% done | ETA(epoch): 1074s + [Epoch 284][20/66] loss=0.734373 avg=0.860316 VRAM=38.6GiB | 94.8% done | ETA(epoch): 883s + [Epoch 284][30/66] loss=0.916294 avg=0.864278 VRAM=38.7GiB | 94.8% done | ETA(epoch): 691s + [Epoch 284][40/66] loss=0.812669 avg=0.857513 VRAM=38.6GiB | 94.9% done | ETA(epoch): 499s + [Epoch 284][50/66] loss=0.696863 avg=0.853931 VRAM=38.7GiB | 94.9% done | ETA(epoch): 307s + [Epoch 284][60/66] loss=0.826561 avg=0.848710 VRAM=38.6GiB | 95.0% done | ETA(epoch): 116s + Train loss: 0.845028 (1270.9s) ETA: 350min + Val loss: 0.842229 [t_0.0-0.2=1.0675 t_0.2-0.4=1.0157 t_0.4-0.6=0.8341 t_0.6-0.8=0.7116 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0284 +[MEM @ epoch 284 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 285/299 (95% done) --- + [Epoch 285][10/66] loss=0.812288 avg=0.851236 VRAM=38.7GiB | 95.1% done | ETA(epoch): 1075s + [Epoch 285][20/66] loss=0.952250 avg=0.877150 VRAM=38.6GiB | 95.1% done | ETA(epoch): 883s + [Epoch 285][30/66] loss=0.777706 avg=0.868910 VRAM=38.7GiB | 95.2% done | ETA(epoch): 691s + [Epoch 285][40/66] loss=0.831796 avg=0.869113 VRAM=38.6GiB | 95.2% done | ETA(epoch): 500s + [Epoch 285][50/66] loss=0.889004 avg=0.866655 VRAM=38.7GiB | 95.3% done | ETA(epoch): 308s + [Epoch 285][60/66] loss=0.857162 avg=0.856834 VRAM=38.6GiB | 95.3% done | ETA(epoch): 115s + Train loss: 0.855138 (1270.0s) ETA: 327min + Val loss: 0.802088 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9841 t_0.4-0.6=0.8413 t_0.6-0.8=0.7060 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 286/299 (95% done) --- + [Epoch 286][10/66] loss=0.888617 avg=0.870018 VRAM=38.7GiB | 95.4% done | ETA(epoch): 1075s + [Epoch 286][20/66] loss=1.047630 avg=0.888208 VRAM=38.6GiB | 95.4% done | ETA(epoch): 883s + [Epoch 286][30/66] loss=0.891484 avg=0.884606 VRAM=38.7GiB | 95.5% done | ETA(epoch): 691s + [Epoch 286][40/66] loss=0.933830 avg=0.873252 VRAM=38.6GiB | 95.5% done | ETA(epoch): 502s + [Epoch 286][50/66] loss=0.873675 avg=0.871920 VRAM=38.7GiB | 95.6% done | ETA(epoch): 309s + [Epoch 286][60/66] loss=0.807618 avg=0.870212 VRAM=38.6GiB | 95.6% done | ETA(epoch): 116s + Train loss: 0.869569 (1272.2s) ETA: 303min + Val loss: 0.822347 [t_0.0-0.2=1.0874 t_0.2-0.4=1.0082 t_0.4-0.6=0.8717 t_0.6-0.8=0.7068 t_0.8-1.0=0.6436] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 287/299 (96% done) --- + [Epoch 287][10/66] loss=0.838554 avg=0.871867 VRAM=38.7GiB | 95.7% done | ETA(epoch): 1077s + [Epoch 287][20/66] loss=0.881357 avg=0.869612 VRAM=38.6GiB | 95.8% done | ETA(epoch): 885s + [Epoch 287][30/66] loss=0.855981 avg=0.869558 VRAM=38.7GiB | 95.8% done | ETA(epoch): 692s + [Epoch 287][40/66] loss=0.740053 avg=0.850899 VRAM=38.6GiB | 95.9% done | ETA(epoch): 500s + [Epoch 287][50/66] loss=0.898729 avg=0.849310 VRAM=38.7GiB | 95.9% done | ETA(epoch): 308s + [Epoch 287][60/66] loss=0.874382 avg=0.851165 VRAM=38.6GiB | 96.0% done | ETA(epoch): 115s + Train loss: 0.851352 (1269.3s) ETA: 280min + Val loss: 0.844504 [t_0.0-0.2=1.0899 t_0.2-0.4=0.9939 t_0.4-0.6=0.8484 t_0.6-0.8=0.6926 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 288/299 (96% done) --- + [Epoch 288][10/66] loss=0.797904 avg=0.809958 VRAM=38.7GiB | 96.1% done | ETA(epoch): 1077s + [Epoch 288][20/66] loss=0.996388 avg=0.871621 VRAM=38.6GiB | 96.1% done | ETA(epoch): 885s + [Epoch 288][30/66] loss=0.835702 avg=0.853681 VRAM=38.7GiB | 96.2% done | ETA(epoch): 692s + [Epoch 288][40/66] loss=0.904861 avg=0.851982 VRAM=38.6GiB | 96.2% done | ETA(epoch): 500s + [Epoch 288][50/66] loss=0.998503 avg=0.864906 VRAM=38.7GiB | 96.3% done | ETA(epoch): 308s + [Epoch 288][60/66] loss=0.988966 avg=0.865949 VRAM=38.6GiB | 96.3% done | ETA(epoch): 115s + Train loss: 0.863186 (1269.6s) ETA: 257min + Val loss: 0.843094 [t_0.0-0.2=1.0903 t_0.2-0.4=1.0070 t_0.4-0.6=0.8179 t_0.6-0.8=0.7090 t_0.8-1.0=0.6379] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 289/299 (96% done) --- + [Epoch 289][10/66] loss=0.931355 avg=0.838301 VRAM=38.7GiB | 96.4% done | ETA(epoch): 1075s + [Epoch 289][20/66] loss=0.768459 avg=0.826503 VRAM=38.6GiB | 96.4% done | ETA(epoch): 888s + [Epoch 289][30/66] loss=0.766940 avg=0.822118 VRAM=38.7GiB | 96.5% done | ETA(epoch): 694s + [Epoch 289][40/66] loss=0.860494 avg=0.829758 VRAM=38.6GiB | 96.5% done | ETA(epoch): 501s + [Epoch 289][50/66] loss=0.814501 avg=0.832148 VRAM=38.7GiB | 96.6% done | ETA(epoch): 308s + [Epoch 289][60/66] loss=0.935802 avg=0.828691 VRAM=38.6GiB | 96.6% done | ETA(epoch): 115s + Train loss: 0.831646 (1270.3s) ETA: 233min + Val loss: 0.837234 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9721 t_0.4-0.6=0.8824 t_0.6-0.8=0.7278 t_0.8-1.0=0.6353] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 290/299 (97% done) --- + [Epoch 290][10/66] loss=0.748409 avg=0.869828 VRAM=38.7GiB | 96.7% done | ETA(epoch): 1077s + [Epoch 290][20/66] loss=0.936431 avg=0.860404 VRAM=38.6GiB | 96.8% done | ETA(epoch): 885s + [Epoch 290][30/66] loss=0.858570 avg=0.859130 VRAM=38.7GiB | 96.8% done | ETA(epoch): 692s + [Epoch 290][40/66] loss=0.764232 avg=0.849396 VRAM=38.6GiB | 96.9% done | ETA(epoch): 500s + [Epoch 290][50/66] loss=0.878094 avg=0.849867 VRAM=38.7GiB | 96.9% done | ETA(epoch): 308s + [Epoch 290][60/66] loss=0.879272 avg=0.852673 VRAM=38.6GiB | 97.0% done | ETA(epoch): 115s + Train loss: 0.858888 (1270.0s) ETA: 210min + Val loss: 0.851807 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9765 t_0.4-0.6=0.8690 t_0.6-0.8=0.7113 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 291/299 (97% done) --- + [Epoch 291][10/66] loss=0.807200 avg=0.834368 VRAM=38.7GiB | 97.1% done | ETA(epoch): 1078s + [Epoch 291][20/66] loss=0.948627 avg=0.861018 VRAM=38.6GiB | 97.1% done | ETA(epoch): 884s + [Epoch 291][30/66] loss=0.830414 avg=0.846439 VRAM=38.7GiB | 97.2% done | ETA(epoch): 692s + [Epoch 291][40/66] loss=0.913777 avg=0.847033 VRAM=38.6GiB | 97.2% done | ETA(epoch): 500s + [Epoch 291][50/66] loss=0.817239 avg=0.848599 VRAM=38.7GiB | 97.3% done | ETA(epoch): 308s + [Epoch 291][60/66] loss=0.952853 avg=0.847024 VRAM=38.6GiB | 97.3% done | ETA(epoch): 115s + Train loss: 0.849714 (1270.3s) ETA: 187min + Val loss: 0.855713 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9872 t_0.4-0.6=0.8675 t_0.6-0.8=0.7062 t_0.8-1.0=0.6296] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 292/299 (97% done) --- + [Epoch 292][10/66] loss=0.914074 avg=0.812109 VRAM=38.7GiB | 97.4% done | ETA(epoch): 1078s + [Epoch 292][20/66] loss=0.886055 avg=0.827928 VRAM=38.6GiB | 97.4% done | ETA(epoch): 885s + [Epoch 292][30/66] loss=0.809664 avg=0.828845 VRAM=38.7GiB | 97.5% done | ETA(epoch): 692s + [Epoch 292][40/66] loss=0.975720 avg=0.838314 VRAM=38.6GiB | 97.5% done | ETA(epoch): 500s + [Epoch 292][50/66] loss=0.963280 avg=0.849924 VRAM=38.7GiB | 97.6% done | ETA(epoch): 307s + [Epoch 292][60/66] loss=0.837118 avg=0.853218 VRAM=38.6GiB | 97.6% done | ETA(epoch): 115s + Train loss: 0.858322 (1270.6s) ETA: 163min + Val loss: 0.863528 [t_0.0-0.2=1.0651 t_0.2-0.4=0.9898 t_0.4-0.6=0.8566 t_0.6-0.8=0.7227 t_0.8-1.0=0.6263] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 293/299 (98% done) --- + [Epoch 293][10/66] loss=0.739120 avg=0.830920 VRAM=38.7GiB | 97.7% done | ETA(epoch): 1078s + [Epoch 293][20/66] loss=0.857761 avg=0.839210 VRAM=38.6GiB | 97.8% done | ETA(epoch): 885s + [Epoch 293][30/66] loss=0.799422 avg=0.833658 VRAM=38.7GiB | 97.8% done | ETA(epoch): 692s + [Epoch 293][40/66] loss=0.925976 avg=0.845420 VRAM=38.6GiB | 97.9% done | ETA(epoch): 500s + [Epoch 293][50/66] loss=0.885847 avg=0.845715 VRAM=38.7GiB | 97.9% done | ETA(epoch): 308s + [Epoch 293][60/66] loss=0.861886 avg=0.847683 VRAM=38.6GiB | 98.0% done | ETA(epoch): 116s + Train loss: 0.844823 (1271.6s) ETA: 140min + Val loss: 0.840480 [t_0.0-0.2=1.0759 t_0.2-0.4=0.9710 t_0.4-0.6=0.8463 t_0.6-0.8=0.7060 t_0.8-1.0=0.6492] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 294/299 (98% done) --- + [Epoch 294][10/66] loss=0.791193 avg=0.856793 VRAM=38.7GiB | 98.1% done | ETA(epoch): 1077s + [Epoch 294][20/66] loss=0.920695 avg=0.882460 VRAM=38.6GiB | 98.1% done | ETA(epoch): 885s + [Epoch 294][30/66] loss=0.833776 avg=0.865913 VRAM=38.7GiB | 98.2% done | ETA(epoch): 693s + [Epoch 294][40/66] loss=0.840918 avg=0.857731 VRAM=38.6GiB | 98.2% done | ETA(epoch): 502s + [Epoch 294][50/66] loss=0.782675 avg=0.856124 VRAM=38.7GiB | 98.3% done | ETA(epoch): 309s + [Epoch 294][60/66] loss=0.914813 avg=0.849559 VRAM=38.6GiB | 98.3% done | ETA(epoch): 116s + Train loss: 0.851471 (1272.4s) ETA: 117min + Val loss: 0.844193 [t_0.0-0.2=1.0906 t_0.2-0.4=1.0236 t_0.4-0.6=0.8394 t_0.6-0.8=0.7103 t_0.8-1.0=0.6427] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 295/299 (98% done) --- + [Epoch 295][10/66] loss=0.800827 avg=0.867514 VRAM=38.7GiB | 98.4% done | ETA(epoch): 1076s + [Epoch 295][20/66] loss=0.911930 avg=0.853573 VRAM=38.6GiB | 98.4% done | ETA(epoch): 885s + [Epoch 295][30/66] loss=0.913446 avg=0.859173 VRAM=38.7GiB | 98.5% done | ETA(epoch): 693s + [Epoch 295][40/66] loss=0.800820 avg=0.859394 VRAM=38.6GiB | 98.5% done | ETA(epoch): 501s + [Epoch 295][50/66] loss=0.922970 avg=0.864127 VRAM=38.7GiB | 98.6% done | ETA(epoch): 308s + [Epoch 295][60/66] loss=0.917433 avg=0.865966 VRAM=38.6GiB | 98.6% done | ETA(epoch): 115s + Train loss: 0.861759 (1270.4s) ETA: 93min + Val loss: 0.858091 [t_0.0-0.2=1.0747 t_0.2-0.4=0.9701 t_0.4-0.6=0.8451 t_0.6-0.8=0.7172 t_0.8-1.0=0.6267] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 296/299 (99% done) --- + [Epoch 296][10/66] loss=0.927037 avg=0.852718 VRAM=38.7GiB | 98.7% done | ETA(epoch): 1078s + [Epoch 296][20/66] loss=0.772143 avg=0.862788 VRAM=38.6GiB | 98.8% done | ETA(epoch): 885s + [Epoch 296][30/66] loss=0.903084 avg=0.861819 VRAM=38.7GiB | 98.8% done | ETA(epoch): 693s + [Epoch 296][40/66] loss=0.848512 avg=0.851894 VRAM=38.6GiB | 98.9% done | ETA(epoch): 500s + [Epoch 296][50/66] loss=0.846683 avg=0.845283 VRAM=38.7GiB | 98.9% done | ETA(epoch): 308s + [Epoch 296][60/66] loss=0.880367 avg=0.846000 VRAM=38.6GiB | 99.0% done | ETA(epoch): 115s + Train loss: 0.849050 (1269.2s) ETA: 70min + Val loss: 0.863780 [t_0.0-0.2=1.0684 t_0.2-0.4=0.9881 t_0.4-0.6=0.8397 t_0.6-0.8=0.7146 t_0.8-1.0=0.6561] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 297/299 (99% done) --- + [Epoch 297][10/66] loss=0.871498 avg=0.853827 VRAM=38.7GiB | 99.1% done | ETA(epoch): 1089s + [Epoch 297][20/66] loss=0.887795 avg=0.870565 VRAM=38.6GiB | 99.1% done | ETA(epoch): 890s + [Epoch 297][30/66] loss=0.768476 avg=0.870272 VRAM=38.7GiB | 99.2% done | ETA(epoch): 695s + [Epoch 297][40/66] loss=0.924783 avg=0.867030 VRAM=38.6GiB | 99.2% done | ETA(epoch): 501s + [Epoch 297][50/66] loss=0.866824 avg=0.866242 VRAM=38.7GiB | 99.3% done | ETA(epoch): 308s + [Epoch 297][60/66] loss=0.811213 avg=0.862817 VRAM=38.6GiB | 99.3% done | ETA(epoch): 116s + Train loss: 0.863200 (1271.1s) ETA: 47min + Val loss: 0.858209 [t_0.0-0.2=1.0774 t_0.2-0.4=1.0044 t_0.4-0.6=0.8309 t_0.6-0.8=0.7213 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 298/299 (99% done) --- + [Epoch 298][10/66] loss=0.871311 avg=0.887356 VRAM=38.7GiB | 99.4% done | ETA(epoch): 1076s + [Epoch 298][20/66] loss=0.700200 avg=0.856846 VRAM=38.6GiB | 99.4% done | ETA(epoch): 884s + [Epoch 298][30/66] loss=0.785431 avg=0.841583 VRAM=38.7GiB | 99.5% done | ETA(epoch): 692s + [Epoch 298][40/66] loss=0.936711 avg=0.836163 VRAM=38.6GiB | 99.5% done | ETA(epoch): 500s + [Epoch 298][50/66] loss=0.823456 avg=0.834777 VRAM=38.7GiB | 99.6% done | ETA(epoch): 308s + [Epoch 298][60/66] loss=0.993028 avg=0.838907 VRAM=38.6GiB | 99.6% done | ETA(epoch): 115s + Train loss: 0.845024 (1269.3s) ETA: 23min + Val loss: 0.853679 [t_0.0-0.2=1.0789 t_0.2-0.4=1.0091 t_0.4-0.6=0.8255 t_0.6-0.8=0.7117 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 299/299 (100% done) --- + [Epoch 299][10/66] loss=0.841613 avg=0.801227 VRAM=38.7GiB | 99.7% done | ETA(epoch): 1074s + [Epoch 299][20/66] loss=0.779123 avg=0.821580 VRAM=38.6GiB | 99.8% done | ETA(epoch): 883s + [Epoch 299][30/66] loss=0.932700 avg=0.835999 VRAM=38.7GiB | 99.8% done | ETA(epoch): 692s + [Epoch 299][40/66] loss=0.836977 avg=0.842654 VRAM=38.6GiB | 99.9% done | ETA(epoch): 500s + [Epoch 299][50/66] loss=0.820199 avg=0.840066 VRAM=38.7GiB | 99.9% done | ETA(epoch): 307s + [Epoch 299][60/66] loss=0.838683 avg=0.843451 VRAM=38.6GiB | 100.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 19799 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 19799 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 19799 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 19799 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 19799 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 19799 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 19799 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 19799 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 19799 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 19799 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.845886 (2314.7s) ETA: 0min + Val loss: 0.859610 [t_0.0-0.2=1.0768 t_0.2-0.4=1.0005 t_0.4-0.6=0.8152 t_0.6-0.8=0.7211 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + [LossCurve WARN] could not save loss curve: No module named 'matplotlib' + +====================================================================== +Training complete. +====================================================================== +wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: +wandb: Run summary: +wandb: adapter_lr 1e-05 +wandb: best_val_loss 0.799 +wandb: cpu_pct 0.8 +wandb: epoch 299 +wandb: final_train_loss 0.84589 +wandb: flops/hdc2a_per_sample_gflops 672.0974 +wandb: flops/total_per_sample_gflops 274113.62308 +wandb: flops/transformer_per_sample_gflops 273441.52568 +wandb: global_step 19793 +wandb: gpu_mem_gib 38.20323 +wandb: +10 ... +wandb: +wandb: View run lora_128_dim_512_H100-train-20260428-190804 at: https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100/runs/eo9bzlri +wandb: View project at: https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100 +wandb: Synced 4 W&B file(s), 110 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260428_190804-eo9bzlri/logs diff --git a/lora_256_dim_1024_H100/train.log b/lora_256_dim_1024_H100/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f90651dd3dcb3fecd45923403cbccbeeca60deee --- /dev/null +++ b/lora_256_dim_1024_H100/train.log @@ -0,0 +1,4566 @@ + +====================================================================== +Command: train_script.py --name lora_256_dim_1024_H100 --batch-size 6 --adapter-lr 1e-3 --num-epochs 300 --lora-lr 1e-4 --adapter-mlp --lora-rank 256 --fusion-dim 1024 --seed 42 +====================================================================== +Run dir : output/lora_256_dim_1024_H100 +Log file: output/lora_256_dim_1024_H100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + +m +wandb: Waiting for wandb.init()... + +m +wandb: setting up run sx2s3t28 (0.3s) + +m +wandb: setting up run sx2s3t28 (0.3s) + +m +wandb: setting up run sx2s3t28 (0.3s) + +m +wandb: setting up run sx2s3t28 (0.3s) + +m +wandb: setting up run sx2s3t28 (0.3s) + +m +wandb: setting up run sx2s3t28 (0.8s) + +m +wandb: setting up run sx2s3t28 (0.8s) + +m +wandb: setting up run sx2s3t28 (0.8s) + +m +wandb: setting up run sx2s3t28 (0.8s) + +m +wandb: setting up run sx2s3t28 (0.8s) + +m +wandb: setting up run sx2s3t28 (1.3s) + +m +wandb: setting up run sx2s3t28 (1.3s) + +m +wandb: setting up run sx2s3t28 (1.3s) + +m +wandb: setting up run sx2s3t28 (1.3s) + +m +wandb: setting up run sx2s3t28 (1.3s) + +m +wandb: setting up run sx2s3t28 (1.8s) + +m +wandb: setting up run sx2s3t28 (1.8s) + +m +wandb: setting up run sx2s3t28 (1.8s) + +m +wandb: setting up run sx2s3t28 (1.8s) + +m +wandb: setting up run sx2s3t28 (1.8s) + +m +wandb: setting up run sx2s3t28 (2.3s) + +m +wandb: setting up run sx2s3t28 (2.3s) + +m +wandb: setting up run sx2s3t28 (2.3s) + +m +wandb: setting up run sx2s3t28 (2.3s) + +m +wandb: setting up run sx2s3t28 (2.3s) + +m +wandb: setting up run sx2s3t28 (2.8s) + +m +wandb: setting up run sx2s3t28 (2.8s) + +m +wandb: setting up run sx2s3t28 (2.8s) + +m +wandb: setting up run sx2s3t28 (2.8s) + +m +wandb: setting up run sx2s3t28 (2.8s) + +m +wandb: setting up run sx2s3t28 (3.3s) + +m +wandb: setting up run sx2s3t28 (3.3s) + +m +wandb: setting up run sx2s3t28 (3.3s) + +m +wandb: setting up run sx2s3t28 (3.3s) + +m +wandb: setting up run sx2s3t28 (3.3s) + +m +wandb: setting up run sx2s3t28 (3.8s) + +m +wandb: setting up run sx2s3t28 (3.8s) + +m +wandb: setting up run sx2s3t28 (3.8s) + +m +wandb: setting up run sx2s3t28 (3.8s) + +m +wandb: setting up run sx2s3t28 (3.8s) + +m +wandb: setting up run sx2s3t28 (4.3s) + +m +wandb: setting up run sx2s3t28 (4.3s) + +m +wandb: setting up run sx2s3t28 (4.3s) + +m +wandb: setting up run sx2s3t28 (4.3s) + +m +wandb: setting up run sx2s3t28 (4.3s) + +m +wandb: setting up run sx2s3t28 (4.8s) + +m +wandb: setting up run sx2s3t28 (4.8s) + +m +wandb: setting up run sx2s3t28 (4.8s) + +m +wandb: setting up run sx2s3t28 (4.8s) + +m +wandb: setting up run sx2s3t28 (4.8s) + +m +wandb: setting up run sx2s3t28 (5.3s) + +m +wandb: setting up run sx2s3t28 (5.3s) + +m +wandb: setting up run sx2s3t28 (5.3s) + +m +wandb: setting up run sx2s3t28 (5.3s) + +m +wandb: setting up run sx2s3t28 (5.3s) + +m +wandb: setting up run sx2s3t28 (5.8s) + +m +wandb: setting up run sx2s3t28 (5.8s) + +m +wandb: setting up run sx2s3t28 (5.8s) + +m +wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260520_185749-sx2s3t28 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_256_dim_1024_H100-train-20260520-185749 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_256_dim_1024_H100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_256_dim_1024_H100/runs/sx2s3t28 + +Run Summary: + name lora_256_dim_1024_H100 + fusion MLP (lightweight) + LoRA ON (rank=256) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_256_dim_1024_H100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 1024 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 300 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_256_dim_1024_H100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 12.9/188.5 GiB (6.8%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.7s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 13.3/188.5 GiB (7.1%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (54.4s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 14.9/188.5 GiB (7.9%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 21.0M params +Control: 0.0M params +Total trainable: 21.0M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=256, alpha=256.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 78.64M + +Parameter Statistics: + HDC²A Adapter: total=21.0M trainable=21.0M + ControlNet (frozen): total=4212.2M LoRA trainable=78.64M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 21.0M + LoRA 78.64M = 99.63M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~4800 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0442, std=0.5195 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.431207 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 47.05 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + depth_encoder.conv_stem.6.weight: 0.008484 + semantic_encoder.conv_stem.6.weight: 0.008362 + W_d.weight: 0.007019 + W_s.weight: 0.006866 + mlp_fusion.3.weight: 0.006470 + Test result: PASSED +[MEM @ after test] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.1/93.1 GiB (40.9%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 1050.088 GFLOPs transformer: 273662.180 GFLOPs total: 274712.268 GFLOPs + per-sample forward FLOPs: 274712.268 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 300 epochs × 66 steps = 19800 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 1980, 3960, 5940, 7920, 9900, 11879, 13859, 15839, 17819, 19799] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/299 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/66] loss=0.960496 avg=0.904887 VRAM=39.0GiB | 0.1% done | ETA(epoch): 6770s + [Epoch 0][20/66] loss=0.930770 avg=0.905419 VRAM=38.9GiB | 0.1% done | ETA(epoch): 3227s + [Epoch 0][30/66] loss=1.056139 avg=0.910574 VRAM=39.0GiB | 0.2% done | ETA(epoch): 1916s + [Epoch 0][40/66] loss=0.840759 avg=0.911978 VRAM=38.9GiB | 0.2% done | ETA(epoch): 1163s + [Epoch 0][50/66] loss=0.906387 avg=0.916072 VRAM=39.0GiB | 0.3% done | ETA(epoch): 635s + [Epoch 0][60/66] loss=0.946830 avg=0.917496 VRAM=38.9GiB | 0.3% done | ETA(epoch): 218s + Train loss: 0.920284 (2293.9s) ETA: 11453min + Val loss: 1.020630 [t_0.0-0.2=1.0888 t_0.2-0.4=1.0668 t_0.4-0.6=0.8858 t_0.6-0.8=0.7747 t_0.8-1.0=1.1118] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 1/299 (0% done) --- + [Epoch 1][10/66] loss=1.029849 avg=0.908017 VRAM=39.0GiB | 0.4% done | ETA(epoch): 1083s + [Epoch 1][20/66] loss=0.917549 avg=0.906482 VRAM=38.9GiB | 0.4% done | ETA(epoch): 890s + [Epoch 1][30/66] loss=0.952678 avg=0.910979 VRAM=39.0GiB | 0.5% done | ETA(epoch): 697s + [Epoch 1][40/66] loss=1.006909 avg=0.914135 VRAM=38.9GiB | 0.5% done | ETA(epoch): 503s + [Epoch 1][50/66] loss=0.894639 avg=0.917182 VRAM=39.0GiB | 0.6% done | ETA(epoch): 310s + [Epoch 1][60/66] loss=0.895016 avg=0.914279 VRAM=38.9GiB | 0.6% done | ETA(epoch): 116s + Train loss: 0.914199 (1277.9s) ETA: 9117min + Val loss: 0.906755 [t_0.0-0.2=1.0837 t_0.2-0.4=1.0264 t_0.4-0.6=0.8771 t_0.6-0.8=0.7668 t_0.8-1.0=0.9006] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 17.8/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 2/299 (1% done) --- + [Epoch 2][10/66] loss=0.983848 avg=0.921258 VRAM=39.0GiB | 0.7% done | ETA(epoch): 1084s + [Epoch 2][20/66] loss=0.942274 avg=0.923600 VRAM=38.9GiB | 0.8% done | ETA(epoch): 890s + [Epoch 2][30/66] loss=0.863820 avg=0.918527 VRAM=39.0GiB | 0.8% done | ETA(epoch): 697s + [Epoch 2][40/66] loss=0.912798 avg=0.917580 VRAM=38.9GiB | 0.9% done | ETA(epoch): 504s + [Epoch 2][50/66] loss=0.901212 avg=0.909034 VRAM=39.0GiB | 0.9% done | ETA(epoch): 310s + [Epoch 2][60/66] loss=0.827146 avg=0.901999 VRAM=38.9GiB | 1.0% done | ETA(epoch): 116s + Train loss: 0.900587 (1278.6s) ETA: 8324min + Val loss: 0.931374 [t_0.0-0.2=1.0936 t_0.2-0.4=1.0167 t_0.4-0.6=0.8311 t_0.6-0.8=0.7565 t_0.8-1.0=0.9217] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 3/299 (1% done) --- + [Epoch 3][10/66] loss=0.812178 avg=0.927982 VRAM=39.0GiB | 1.1% done | ETA(epoch): 1099s + [Epoch 3][20/66] loss=0.824597 avg=0.914004 VRAM=38.9GiB | 1.1% done | ETA(epoch): 896s + [Epoch 3][30/66] loss=0.867018 avg=0.909529 VRAM=39.0GiB | 1.2% done | ETA(epoch): 700s + [Epoch 3][40/66] loss=0.991074 avg=0.910376 VRAM=38.9GiB | 1.2% done | ETA(epoch): 505s + [Epoch 3][50/66] loss=0.876312 avg=0.907049 VRAM=39.0GiB | 1.3% done | ETA(epoch): 310s + [Epoch 3][60/66] loss=0.841781 avg=0.901090 VRAM=38.9GiB | 1.3% done | ETA(epoch): 116s + Train loss: 0.897321 (1280.0s) ETA: 7918min + Val loss: 0.886077 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9768 t_0.4-0.6=0.8579 t_0.6-0.8=0.7737 t_0.8-1.0=0.9187] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 4/299 (1% done) --- + [Epoch 4][10/66] loss=0.869948 avg=0.864543 VRAM=39.0GiB | 1.4% done | ETA(epoch): 1088s + [Epoch 4][20/66] loss=0.902650 avg=0.873414 VRAM=38.9GiB | 1.4% done | ETA(epoch): 893s + [Epoch 4][30/66] loss=0.905774 avg=0.884700 VRAM=39.0GiB | 1.5% done | ETA(epoch): 698s + [Epoch 4][40/66] loss=0.835370 avg=0.879331 VRAM=38.9GiB | 1.5% done | ETA(epoch): 504s + [Epoch 4][50/66] loss=0.865286 avg=0.889609 VRAM=39.0GiB | 1.6% done | ETA(epoch): 310s + [Epoch 4][60/66] loss=0.864625 avg=0.879462 VRAM=38.9GiB | 1.6% done | ETA(epoch): 116s + Train loss: 0.878582 (1281.9s) ETA: 7669min + Val loss: 0.938194 [t_0.0-0.2=1.0822 t_0.2-0.4=1.0152 t_0.4-0.6=0.8806 t_0.6-0.8=0.7372 t_0.8-1.0=0.9327] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 5/299 (2% done) --- + [Epoch 5][10/66] loss=0.774916 avg=0.874291 VRAM=39.0GiB | 1.7% done | ETA(epoch): 1083s + [Epoch 5][20/66] loss=0.909404 avg=0.872365 VRAM=38.9GiB | 1.8% done | ETA(epoch): 890s + [Epoch 5][30/66] loss=0.829275 avg=0.873166 VRAM=39.0GiB | 1.8% done | ETA(epoch): 696s + [Epoch 5][40/66] loss=0.965062 avg=0.886540 VRAM=38.9GiB | 1.9% done | ETA(epoch): 503s + [Epoch 5][50/66] loss=0.829771 avg=0.881359 VRAM=39.0GiB | 1.9% done | ETA(epoch): 310s + [Epoch 5][60/66] loss=0.845142 avg=0.881310 VRAM=38.9GiB | 2.0% done | ETA(epoch): 116s + Train loss: 0.885467 (1277.3s) ETA: 7490min + Val loss: 0.895680 [t_0.0-0.2=1.0855 t_0.2-0.4=0.9816 t_0.4-0.6=0.8354 t_0.6-0.8=0.7675 t_0.8-1.0=0.8586] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 6/299 (2% done) --- + [Epoch 6][10/66] loss=0.826390 avg=0.896742 VRAM=39.0GiB | 2.1% done | ETA(epoch): 1084s + [Epoch 6][20/66] loss=0.924840 avg=0.900427 VRAM=38.9GiB | 2.1% done | ETA(epoch): 893s + [Epoch 6][30/66] loss=0.976652 avg=0.901363 VRAM=39.0GiB | 2.2% done | ETA(epoch): 699s + [Epoch 6][40/66] loss=0.993579 avg=0.900548 VRAM=38.9GiB | 2.2% done | ETA(epoch): 505s + [Epoch 6][50/66] loss=0.875573 avg=0.904832 VRAM=39.0GiB | 2.3% done | ETA(epoch): 313s + [Epoch 6][60/66] loss=0.926337 avg=0.900020 VRAM=38.9GiB | 2.3% done | ETA(epoch): 117s + Train loss: 0.899316 (1287.7s) ETA: 7364min + Val loss: 0.913713 [t_0.0-0.2=1.0690 t_0.2-0.4=1.0147 t_0.4-0.6=0.8419 t_0.6-0.8=0.7596 t_0.8-1.0=0.8845] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0006 +[MEM @ epoch 6 end] RAM: 17.8/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 7/299 (2% done) --- + [Epoch 7][10/66] loss=0.933939 avg=0.907425 VRAM=39.0GiB | 2.4% done | ETA(epoch): 1092s + [Epoch 7][20/66] loss=0.823888 avg=0.887865 VRAM=38.9GiB | 2.4% done | ETA(epoch): 897s + [Epoch 7][30/66] loss=0.848215 avg=0.891903 VRAM=39.0GiB | 2.5% done | ETA(epoch): 701s + [Epoch 7][40/66] loss=0.867900 avg=0.888901 VRAM=38.9GiB | 2.5% done | ETA(epoch): 506s + [Epoch 7][50/66] loss=0.872299 avg=0.886684 VRAM=39.0GiB | 2.6% done | ETA(epoch): 311s + [Epoch 7][60/66] loss=0.909610 avg=0.889455 VRAM=38.9GiB | 2.6% done | ETA(epoch): 117s + Train loss: 0.890138 (1281.8s) ETA: 7259min + Val loss: 0.863849 [t_0.0-0.2=1.0861 t_0.2-0.4=0.9808 t_0.4-0.6=0.8654 t_0.6-0.8=0.7507 t_0.8-1.0=0.7678] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0007 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 8/299 (3% done) --- + [Epoch 8][10/66] loss=0.848540 avg=0.860653 VRAM=39.0GiB | 2.7% done | ETA(epoch): 1085s + [Epoch 8][20/66] loss=0.911942 avg=0.868019 VRAM=38.9GiB | 2.8% done | ETA(epoch): 891s + [Epoch 8][30/66] loss=0.916514 avg=0.866343 VRAM=39.0GiB | 2.8% done | ETA(epoch): 697s + [Epoch 8][40/66] loss=0.927426 avg=0.878409 VRAM=38.9GiB | 2.9% done | ETA(epoch): 504s + [Epoch 8][50/66] loss=0.749645 avg=0.876806 VRAM=39.0GiB | 2.9% done | ETA(epoch): 310s + [Epoch 8][60/66] loss=0.797897 avg=0.875845 VRAM=38.9GiB | 3.0% done | ETA(epoch): 116s + Train loss: 0.876492 (1279.0s) ETA: 7172min + Val loss: 0.880301 [t_0.0-0.2=1.0814 t_0.2-0.4=1.0235 t_0.4-0.6=0.8606 t_0.6-0.8=0.7377 t_0.8-1.0=0.7733] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 9/299 (3% done) --- + [Epoch 9][10/66] loss=0.943021 avg=0.888231 VRAM=39.0GiB | 3.1% done | ETA(epoch): 1085s + [Epoch 9][20/66] loss=0.917974 avg=0.876329 VRAM=38.9GiB | 3.1% done | ETA(epoch): 892s + [Epoch 9][30/66] loss=0.869246 avg=0.879723 VRAM=39.0GiB | 3.2% done | ETA(epoch): 698s + [Epoch 9][40/66] loss=1.026034 avg=0.880904 VRAM=38.9GiB | 3.2% done | ETA(epoch): 504s + [Epoch 9][50/66] loss=0.736823 avg=0.876356 VRAM=39.0GiB | 3.3% done | ETA(epoch): 310s + [Epoch 9][60/66] loss=0.822878 avg=0.876145 VRAM=38.9GiB | 3.3% done | ETA(epoch): 116s + Train loss: 0.882924 (1280.2s) ETA: 7098min + Val loss: 0.889496 [t_0.0-0.2=1.0742 t_0.2-0.4=1.0222 t_0.4-0.6=0.8584 t_0.6-0.8=0.7199 t_0.8-1.0=0.7504] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 17.8/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 10/299 (3% done) --- + [Epoch 10][10/66] loss=0.842095 avg=0.898244 VRAM=39.0GiB | 3.4% done | ETA(epoch): 1098s + [Epoch 10][20/66] loss=0.811956 avg=0.880211 VRAM=38.9GiB | 3.4% done | ETA(epoch): 898s + [Epoch 10][30/66] loss=0.941868 avg=0.877602 VRAM=39.0GiB | 3.5% done | ETA(epoch): 702s + [Epoch 10][40/66] loss=0.906036 avg=0.877907 VRAM=38.9GiB | 3.5% done | ETA(epoch): 506s + [Epoch 10][50/66] loss=0.849926 avg=0.875243 VRAM=39.0GiB | 3.6% done | ETA(epoch): 311s + [Epoch 10][60/66] loss=0.804252 avg=0.874645 VRAM=38.9GiB | 3.6% done | ETA(epoch): 117s + Train loss: 0.875524 (1283.0s) ETA: 7034min + Val loss: 0.889589 [t_0.0-0.2=1.0771 t_0.2-0.4=1.0193 t_0.4-0.6=0.8452 t_0.6-0.8=0.7236 t_0.8-1.0=0.7926] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0010 +[MEM @ epoch 10 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 11/299 (4% done) --- + [Epoch 11][10/66] loss=0.804567 avg=0.870049 VRAM=39.0GiB | 3.7% done | ETA(epoch): 1095s + [Epoch 11][20/66] loss=1.005376 avg=0.878979 VRAM=38.9GiB | 3.8% done | ETA(epoch): 896s + [Epoch 11][30/66] loss=0.798158 avg=0.873399 VRAM=39.0GiB | 3.8% done | ETA(epoch): 700s + [Epoch 11][40/66] loss=0.826167 avg=0.871424 VRAM=38.9GiB | 3.9% done | ETA(epoch): 505s + [Epoch 11][50/66] loss=0.880435 avg=0.870686 VRAM=39.0GiB | 3.9% done | ETA(epoch): 311s + [Epoch 11][60/66] loss=0.887924 avg=0.869534 VRAM=38.9GiB | 4.0% done | ETA(epoch): 116s + Train loss: 0.867791 (1281.1s) ETA: 6976min + Val loss: 0.885562 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9959 t_0.4-0.6=0.8309 t_0.6-0.8=0.7480 t_0.8-1.0=0.7662] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 12/299 (4% done) --- + [Epoch 12][10/66] loss=0.892268 avg=0.891487 VRAM=39.0GiB | 4.1% done | ETA(epoch): 1082s + [Epoch 12][20/66] loss=0.903638 avg=0.908595 VRAM=38.9GiB | 4.1% done | ETA(epoch): 890s + [Epoch 12][30/66] loss=0.859808 avg=0.904664 VRAM=39.0GiB | 4.2% done | ETA(epoch): 697s + [Epoch 12][40/66] loss=0.876076 avg=0.894481 VRAM=38.9GiB | 4.2% done | ETA(epoch): 503s + [Epoch 12][50/66] loss=0.777438 avg=0.890615 VRAM=39.0GiB | 4.3% done | ETA(epoch): 310s + [Epoch 12][60/66] loss=0.846110 avg=0.885968 VRAM=38.9GiB | 4.3% done | ETA(epoch): 117s + Train loss: 0.881346 (1284.1s) ETA: 6925min + Val loss: 0.882182 [t_0.0-0.2=1.0776 t_0.2-0.4=1.0153 t_0.4-0.6=0.8602 t_0.6-0.8=0.7284 t_0.8-1.0=0.7533] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 16.3/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 13/299 (4% done) --- + [Epoch 13][10/66] loss=0.898674 avg=0.896899 VRAM=39.0GiB | 4.4% done | ETA(epoch): 1084s + [Epoch 13][20/66] loss=0.852682 avg=0.894645 VRAM=38.9GiB | 4.4% done | ETA(epoch): 890s + [Epoch 13][30/66] loss=0.857858 avg=0.898389 VRAM=39.0GiB | 4.5% done | ETA(epoch): 697s + [Epoch 13][40/66] loss=0.975151 avg=0.898156 VRAM=38.9GiB | 4.5% done | ETA(epoch): 504s + [Epoch 13][50/66] loss=0.777452 avg=0.890695 VRAM=39.0GiB | 4.6% done | ETA(epoch): 312s + [Epoch 13][60/66] loss=0.963370 avg=0.889191 VRAM=38.9GiB | 4.6% done | ETA(epoch): 117s + Train loss: 0.887246 (1286.8s) ETA: 6879min + Val loss: 0.869172 [t_0.0-0.2=1.0843 t_0.2-0.4=1.0121 t_0.4-0.6=0.8509 t_0.6-0.8=0.7401 t_0.8-1.0=0.7400] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 14/299 (5% done) --- + [Epoch 14][10/66] loss=0.871692 avg=0.871648 VRAM=39.0GiB | 4.7% done | ETA(epoch): 1087s + [Epoch 14][20/66] loss=0.933341 avg=0.901269 VRAM=38.9GiB | 4.8% done | ETA(epoch): 896s + [Epoch 14][30/66] loss=0.915147 avg=0.891001 VRAM=39.0GiB | 4.8% done | ETA(epoch): 701s + [Epoch 14][40/66] loss=0.903574 avg=0.889556 VRAM=38.9GiB | 4.9% done | ETA(epoch): 508s + [Epoch 14][50/66] loss=0.855366 avg=0.879424 VRAM=39.0GiB | 4.9% done | ETA(epoch): 312s + [Epoch 14][60/66] loss=0.944263 avg=0.880202 VRAM=38.9GiB | 5.0% done | ETA(epoch): 117s + Train loss: 0.878737 (1285.2s) ETA: 6835min + Val loss: 0.877635 [t_0.0-0.2=1.0742 t_0.2-0.4=1.0156 t_0.4-0.6=0.8850 t_0.6-0.8=0.7011 t_0.8-1.0=0.7349] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0014 + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 15/299 (5% done) --- + [Epoch 15][10/66] loss=0.836180 avg=0.872667 VRAM=39.0GiB | 5.1% done | ETA(epoch): 1090s + [Epoch 15][20/66] loss=0.816003 avg=0.876180 VRAM=38.9GiB | 5.1% done | ETA(epoch): 892s + [Epoch 15][30/66] loss=0.803651 avg=0.872439 VRAM=39.0GiB | 5.2% done | ETA(epoch): 708s + [Epoch 15][40/66] loss=0.809304 avg=0.874903 VRAM=38.9GiB | 5.2% done | ETA(epoch): 509s + [Epoch 15][50/66] loss=0.993142 avg=0.878784 VRAM=39.0GiB | 5.3% done | ETA(epoch): 313s + [Epoch 15][60/66] loss=0.829750 avg=0.874825 VRAM=38.9GiB | 5.3% done | ETA(epoch): 117s + Train loss: 0.873789 (1288.8s) ETA: 6796min + Val loss: 0.880297 [t_0.0-0.2=1.0701 t_0.2-0.4=1.0151 t_0.4-0.6=0.8547 t_0.6-0.8=0.6988 t_0.8-1.0=0.7624] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 16/299 (5% done) --- + [Epoch 16][10/66] loss=0.850798 avg=0.894896 VRAM=39.0GiB | 5.4% done | ETA(epoch): 1085s + [Epoch 16][20/66] loss=0.954378 avg=0.892048 VRAM=38.9GiB | 5.4% done | ETA(epoch): 893s + [Epoch 16][30/66] loss=0.863645 avg=0.887779 VRAM=39.0GiB | 5.5% done | ETA(epoch): 698s + [Epoch 16][40/66] loss=1.003433 avg=0.884784 VRAM=38.9GiB | 5.5% done | ETA(epoch): 504s + [Epoch 16][50/66] loss=0.898502 avg=0.887453 VRAM=39.0GiB | 5.6% done | ETA(epoch): 310s + [Epoch 16][60/66] loss=0.795531 avg=0.886213 VRAM=38.9GiB | 5.6% done | ETA(epoch): 116s + Train loss: 0.884091 (1279.2s) ETA: 6755min + Val loss: 0.883255 [t_0.0-0.2=1.0803 t_0.2-0.4=0.9899 t_0.4-0.6=0.8626 t_0.6-0.8=0.7332 t_0.8-1.0=0.7158] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 17/299 (6% done) --- + [Epoch 17][10/66] loss=0.853082 avg=0.847765 VRAM=39.0GiB | 5.7% done | ETA(epoch): 1110s + [Epoch 17][20/66] loss=0.919747 avg=0.861841 VRAM=38.9GiB | 5.8% done | ETA(epoch): 901s + [Epoch 17][30/66] loss=0.867698 avg=0.869061 VRAM=39.0GiB | 5.8% done | ETA(epoch): 703s + [Epoch 17][40/66] loss=0.746591 avg=0.874448 VRAM=38.9GiB | 5.9% done | ETA(epoch): 506s + [Epoch 17][50/66] loss=0.911654 avg=0.869557 VRAM=39.0GiB | 5.9% done | ETA(epoch): 311s + [Epoch 17][60/66] loss=0.986303 avg=0.873528 VRAM=38.9GiB | 6.0% done | ETA(epoch): 117s + Train loss: 0.875527 (1282.7s) ETA: 6718min + Val loss: 0.887950 [t_0.0-0.2=1.0758 t_0.2-0.4=1.0136 t_0.4-0.6=0.8718 t_0.6-0.8=0.7438 t_0.8-1.0=0.7297] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 18/299 (6% done) --- + [Epoch 18][10/66] loss=0.900566 avg=0.857527 VRAM=39.0GiB | 6.1% done | ETA(epoch): 1084s + [Epoch 18][20/66] loss=0.809163 avg=0.873702 VRAM=38.9GiB | 6.1% done | ETA(epoch): 891s + [Epoch 18][30/66] loss=0.867430 avg=0.878283 VRAM=39.0GiB | 6.2% done | ETA(epoch): 697s + [Epoch 18][40/66] loss=0.813247 avg=0.869066 VRAM=38.9GiB | 6.2% done | ETA(epoch): 505s + [Epoch 18][50/66] loss=0.841257 avg=0.860803 VRAM=39.0GiB | 6.3% done | ETA(epoch): 311s + [Epoch 18][60/66] loss=0.898552 avg=0.856007 VRAM=38.9GiB | 6.3% done | ETA(epoch): 116s + Train loss: 0.858063 (1283.5s) ETA: 6682min + Val loss: 0.908944 [t_0.0-0.2=1.0846 t_0.2-0.4=1.0437 t_0.4-0.6=0.8531 t_0.6-0.8=0.7225 t_0.8-1.0=0.6988] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 19/299 (6% done) --- + [Epoch 19][10/66] loss=0.922899 avg=0.886303 VRAM=39.0GiB | 6.4% done | ETA(epoch): 1086s + [Epoch 19][20/66] loss=0.858232 avg=0.883401 VRAM=38.9GiB | 6.4% done | ETA(epoch): 893s + [Epoch 19][30/66] loss=0.981754 avg=0.882535 VRAM=39.0GiB | 6.5% done | ETA(epoch): 701s + [Epoch 19][40/66] loss=0.894578 avg=0.887234 VRAM=38.9GiB | 6.5% done | ETA(epoch): 506s + [Epoch 19][50/66] loss=0.869473 avg=0.886090 VRAM=39.0GiB | 6.6% done | ETA(epoch): 311s + [Epoch 19][60/66] loss=0.949640 avg=0.883118 VRAM=38.9GiB | 6.6% done | ETA(epoch): 117s + Train loss: 0.886352 (1283.6s) ETA: 6647min + Val loss: 0.887697 [t_0.0-0.2=1.0617 t_0.2-0.4=1.0214 t_0.4-0.6=0.8566 t_0.6-0.8=0.7489 t_0.8-1.0=0.7087] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 20/299 (7% done) --- + [Epoch 20][10/66] loss=0.747296 avg=0.864585 VRAM=39.0GiB | 6.7% done | ETA(epoch): 1083s + [Epoch 20][20/66] loss=0.941617 avg=0.865006 VRAM=38.9GiB | 6.8% done | ETA(epoch): 892s + [Epoch 20][30/66] loss=0.904612 avg=0.865483 VRAM=39.0GiB | 6.8% done | ETA(epoch): 698s + [Epoch 20][40/66] loss=0.761008 avg=0.865663 VRAM=38.9GiB | 6.9% done | ETA(epoch): 504s + [Epoch 20][50/66] loss=0.917383 avg=0.872581 VRAM=39.0GiB | 6.9% done | ETA(epoch): 311s + [Epoch 20][60/66] loss=0.792323 avg=0.869579 VRAM=38.9GiB | 7.0% done | ETA(epoch): 117s + Train loss: 0.870453 (1282.5s) ETA: 6613min + Val loss: 0.885173 [t_0.0-0.2=1.0784 t_0.2-0.4=1.0063 t_0.4-0.6=0.8508 t_0.6-0.8=0.7518 t_0.8-1.0=0.7042] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 21/299 (7% done) --- + [Epoch 21][10/66] loss=0.821887 avg=0.862079 VRAM=39.0GiB | 7.1% done | ETA(epoch): 1085s + [Epoch 21][20/66] loss=0.797199 avg=0.851830 VRAM=38.9GiB | 7.1% done | ETA(epoch): 894s + [Epoch 21][30/66] loss=0.837849 avg=0.856183 VRAM=39.0GiB | 7.2% done | ETA(epoch): 699s + [Epoch 21][40/66] loss=0.761577 avg=0.850210 VRAM=38.9GiB | 7.2% done | ETA(epoch): 505s + [Epoch 21][50/66] loss=0.826577 avg=0.852739 VRAM=39.0GiB | 7.3% done | ETA(epoch): 311s + [Epoch 21][60/66] loss=0.924396 avg=0.853067 VRAM=38.9GiB | 7.3% done | ETA(epoch): 116s + Train loss: 0.851308 (1280.7s) ETA: 6580min + Val loss: 0.877188 [t_0.0-0.2=1.0641 t_0.2-0.4=0.9841 t_0.4-0.6=0.8723 t_0.6-0.8=0.7206 t_0.8-1.0=0.7009] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 16.3/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 22/299 (7% done) --- + [Epoch 22][10/66] loss=0.790767 avg=0.825075 VRAM=39.0GiB | 7.4% done | ETA(epoch): 1082s + [Epoch 22][20/66] loss=0.953616 avg=0.857609 VRAM=38.9GiB | 7.4% done | ETA(epoch): 890s + [Epoch 22][30/66] loss=0.853681 avg=0.854056 VRAM=39.0GiB | 7.5% done | ETA(epoch): 698s + [Epoch 22][40/66] loss=0.832731 avg=0.864257 VRAM=38.9GiB | 7.5% done | ETA(epoch): 504s + [Epoch 22][50/66] loss=0.947649 avg=0.868493 VRAM=39.0GiB | 7.6% done | ETA(epoch): 310s + [Epoch 22][60/66] loss=0.851869 avg=0.866572 VRAM=38.9GiB | 7.6% done | ETA(epoch): 116s + Train loss: 0.870622 (1279.1s) ETA: 6547min + Val loss: 0.878845 [t_0.0-0.2=1.0879 t_0.2-0.4=1.0072 t_0.4-0.6=0.8292 t_0.6-0.8=0.7433 t_0.8-1.0=0.6956] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 23/299 (8% done) --- + [Epoch 23][10/66] loss=0.888730 avg=0.872534 VRAM=39.0GiB | 7.7% done | ETA(epoch): 1083s + [Epoch 23][20/66] loss=0.930789 avg=0.872139 VRAM=38.9GiB | 7.8% done | ETA(epoch): 897s + [Epoch 23][30/66] loss=0.847068 avg=0.862818 VRAM=39.0GiB | 7.8% done | ETA(epoch): 701s + [Epoch 23][40/66] loss=0.809318 avg=0.859497 VRAM=38.9GiB | 7.9% done | ETA(epoch): 506s + [Epoch 23][50/66] loss=0.893061 avg=0.858801 VRAM=39.0GiB | 7.9% done | ETA(epoch): 311s + [Epoch 23][60/66] loss=0.962948 avg=0.864924 VRAM=38.9GiB | 8.0% done | ETA(epoch): 117s + Train loss: 0.869211 (1283.3s) ETA: 6516min + Val loss: 0.899865 [t_0.0-0.2=1.0654 t_0.2-0.4=1.0047 t_0.4-0.6=0.8447 t_0.6-0.8=0.7447 t_0.8-1.0=0.6786] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 24/299 (8% done) --- + [Epoch 24][10/66] loss=0.901107 avg=0.874584 VRAM=39.0GiB | 8.1% done | ETA(epoch): 1123s + [Epoch 24][20/66] loss=0.857800 avg=0.866630 VRAM=38.9GiB | 8.1% done | ETA(epoch): 907s + [Epoch 24][30/66] loss=0.808626 avg=0.859655 VRAM=39.0GiB | 8.2% done | ETA(epoch): 706s + [Epoch 24][40/66] loss=0.865225 avg=0.859865 VRAM=38.9GiB | 8.2% done | ETA(epoch): 508s + [Epoch 24][50/66] loss=0.791441 avg=0.861299 VRAM=39.0GiB | 8.3% done | ETA(epoch): 312s + [Epoch 24][60/66] loss=0.840735 avg=0.865717 VRAM=38.9GiB | 8.3% done | ETA(epoch): 117s + Train loss: 0.866649 (1284.9s) ETA: 6486min + Val loss: 0.887356 [t_0.0-0.2=1.0895 t_0.2-0.4=0.9796 t_0.4-0.6=0.8307 t_0.6-0.8=0.7252 t_0.8-1.0=0.6942] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 16.3/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 25/299 (8% done) --- + [Epoch 25][10/66] loss=0.883843 avg=0.866852 VRAM=39.0GiB | 8.4% done | ETA(epoch): 1098s + [Epoch 25][20/66] loss=0.889293 avg=0.855081 VRAM=38.9GiB | 8.4% done | ETA(epoch): 898s + [Epoch 25][30/66] loss=0.825581 avg=0.864781 VRAM=39.0GiB | 8.5% done | ETA(epoch): 702s + [Epoch 25][40/66] loss=0.892555 avg=0.873618 VRAM=38.9GiB | 8.5% done | ETA(epoch): 506s + [Epoch 25][50/66] loss=0.812659 avg=0.869410 VRAM=39.0GiB | 8.6% done | ETA(epoch): 311s + [Epoch 25][60/66] loss=0.752444 avg=0.863363 VRAM=38.9GiB | 8.6% done | ETA(epoch): 117s + Train loss: 0.859766 (1283.5s) ETA: 6456min + Val loss: 0.874730 [t_0.0-0.2=1.0701 t_0.2-0.4=1.0136 t_0.4-0.6=0.8845 t_0.6-0.8=0.7245 t_0.8-1.0=0.7029] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 16.3/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 26/299 (9% done) --- + [Epoch 26][10/66] loss=0.875078 avg=0.890119 VRAM=39.0GiB | 8.7% done | ETA(epoch): 1085s + [Epoch 26][20/66] loss=0.936797 avg=0.899335 VRAM=38.9GiB | 8.8% done | ETA(epoch): 893s + [Epoch 26][30/66] loss=0.768418 avg=0.881121 VRAM=39.0GiB | 8.8% done | ETA(epoch): 698s + [Epoch 26][40/66] loss=0.854100 avg=0.874197 VRAM=38.9GiB | 8.9% done | ETA(epoch): 505s + [Epoch 26][50/66] loss=0.895918 avg=0.874902 VRAM=39.0GiB | 8.9% done | ETA(epoch): 311s + [Epoch 26][60/66] loss=0.804042 avg=0.871327 VRAM=38.9GiB | 9.0% done | ETA(epoch): 117s + Train loss: 0.868628 (1283.0s) ETA: 6427min + Val loss: 0.873788 [t_0.0-0.2=1.0685 t_0.2-0.4=1.0210 t_0.4-0.6=0.8652 t_0.6-0.8=0.7224 t_0.8-1.0=0.6785] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 27/299 (9% done) --- + [Epoch 27][10/66] loss=0.850466 avg=0.894246 VRAM=39.0GiB | 9.1% done | ETA(epoch): 1083s + [Epoch 27][20/66] loss=0.834585 avg=0.878492 VRAM=38.9GiB | 9.1% done | ETA(epoch): 895s + [Epoch 27][30/66] loss=0.829857 avg=0.868587 VRAM=39.0GiB | 9.2% done | ETA(epoch): 699s + [Epoch 27][40/66] loss=0.920197 avg=0.872399 VRAM=38.9GiB | 9.2% done | ETA(epoch): 504s + [Epoch 27][50/66] loss=0.742510 avg=0.870893 VRAM=39.0GiB | 9.3% done | ETA(epoch): 310s + [Epoch 27][60/66] loss=0.999042 avg=0.869671 VRAM=38.9GiB | 9.3% done | ETA(epoch): 116s + Train loss: 0.870961 (1279.4s) ETA: 6397min + Val loss: 0.865959 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0133 t_0.4-0.6=0.8317 t_0.6-0.8=0.7291 t_0.8-1.0=0.6991] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 16.3/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 28/299 (9% done) --- + [Epoch 28][10/66] loss=1.000633 avg=0.896305 VRAM=39.0GiB | 9.4% done | ETA(epoch): 1086s + [Epoch 28][20/66] loss=0.946692 avg=0.890869 VRAM=38.9GiB | 9.4% done | ETA(epoch): 892s + [Epoch 28][30/66] loss=0.804984 avg=0.878275 VRAM=39.0GiB | 9.5% done | ETA(epoch): 698s + [Epoch 28][40/66] loss=0.963226 avg=0.869302 VRAM=38.9GiB | 9.5% done | ETA(epoch): 505s + [Epoch 28][50/66] loss=0.789010 avg=0.869692 VRAM=39.0GiB | 9.6% done | ETA(epoch): 311s + [Epoch 28][60/66] loss=0.953493 avg=0.878281 VRAM=38.9GiB | 9.6% done | ETA(epoch): 116s + Train loss: 0.882791 (1281.3s) ETA: 6368min + Val loss: 0.859286 [t_0.0-0.2=1.0883 t_0.2-0.4=0.9524 t_0.4-0.6=0.8392 t_0.6-0.8=0.7341 t_0.8-1.0=0.7118] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0028 (BEST) + Deleted old checkpoint: checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 29/299 (10% done) --- + [Epoch 29][10/66] loss=0.900545 avg=0.831985 VRAM=39.0GiB | 9.7% done | ETA(epoch): 1090s + [Epoch 29][20/66] loss=0.826160 avg=0.850829 VRAM=38.9GiB | 9.8% done | ETA(epoch): 899s + [Epoch 29][30/66] loss=0.853474 avg=0.874265 VRAM=39.0GiB | 9.8% done | ETA(epoch): 704s + [Epoch 29][40/66] loss=0.859934 avg=0.865908 VRAM=38.9GiB | 9.9% done | ETA(epoch): 507s + [Epoch 29][50/66] loss=0.949960 avg=0.871688 VRAM=39.0GiB | 9.9% done | ETA(epoch): 312s + [Epoch 29][60/66] loss=0.950803 avg=0.865531 VRAM=38.9GiB | 10.0% done | ETA(epoch): 117s + Train loss: 0.862594 (1284.6s) ETA: 6340min + Val loss: 0.871624 [t_0.0-0.2=1.0770 t_0.2-0.4=0.9884 t_0.4-0.6=0.8572 t_0.6-0.8=0.7634 t_0.8-1.0=0.7182] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 16.3/188.5 GiB (8.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 30/299 (10% done) --- + [MilestoneVis] train_0 step 1980 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 1980 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 1980 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 1980 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 1980 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 1980 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 1980 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 1980 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 1980 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 1980 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 30][10/66] loss=0.963869 avg=0.878623 VRAM=39.0GiB | 10.1% done | ETA(epoch): 6832s + [Epoch 30][20/66] loss=0.779466 avg=0.861891 VRAM=38.9GiB | 10.1% done | ETA(epoch): 3251s + [Epoch 30][30/66] loss=0.990639 avg=0.847434 VRAM=39.0GiB | 10.2% done | ETA(epoch): 1933s + [Epoch 30][40/66] loss=0.988655 avg=0.857502 VRAM=38.9GiB | 10.2% done | ETA(epoch): 1173s + [Epoch 30][50/66] loss=0.877614 avg=0.861265 VRAM=39.0GiB | 10.3% done | ETA(epoch): 640s + [Epoch 30][60/66] loss=0.817159 avg=0.867299 VRAM=38.9GiB | 10.3% done | ETA(epoch): 219s + Train loss: 0.869499 (2309.9s) ETA: 6461min + Val loss: 0.867900 [t_0.0-0.2=1.0723 t_0.2-0.4=1.0169 t_0.4-0.6=0.8255 t_0.6-0.8=0.7356 t_0.8-1.0=0.7047] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 31/299 (10% done) --- + [Epoch 31][10/66] loss=0.909524 avg=0.851055 VRAM=39.0GiB | 10.4% done | ETA(epoch): 1082s + [Epoch 31][20/66] loss=0.848256 avg=0.875390 VRAM=38.9GiB | 10.4% done | ETA(epoch): 895s + [Epoch 31][30/66] loss=0.754916 avg=0.872980 VRAM=39.0GiB | 10.5% done | ETA(epoch): 702s + [Epoch 31][40/66] loss=0.794942 avg=0.867393 VRAM=38.9GiB | 10.5% done | ETA(epoch): 506s + [Epoch 31][50/66] loss=0.871458 avg=0.864025 VRAM=39.0GiB | 10.6% done | ETA(epoch): 312s + [Epoch 31][60/66] loss=0.909529 avg=0.864233 VRAM=38.9GiB | 10.6% done | ETA(epoch): 117s + Train loss: 0.864967 (1282.3s) ETA: 6428min + Val loss: 0.855948 [t_0.0-0.2=1.0778 t_0.2-0.4=0.9571 t_0.4-0.6=0.8922 t_0.6-0.8=0.7345 t_0.8-1.0=0.6829] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0031 (BEST) + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 32/299 (11% done) --- + [Epoch 32][10/66] loss=0.977787 avg=0.877909 VRAM=39.0GiB | 10.7% done | ETA(epoch): 1097s + [Epoch 32][20/66] loss=0.886557 avg=0.877289 VRAM=38.9GiB | 10.8% done | ETA(epoch): 897s + [Epoch 32][30/66] loss=0.892288 avg=0.879551 VRAM=39.0GiB | 10.8% done | ETA(epoch): 701s + [Epoch 32][40/66] loss=0.984636 avg=0.885443 VRAM=38.9GiB | 10.9% done | ETA(epoch): 505s + [Epoch 32][50/66] loss=0.872755 avg=0.877004 VRAM=39.0GiB | 10.9% done | ETA(epoch): 311s + [Epoch 32][60/66] loss=0.815908 avg=0.874490 VRAM=38.9GiB | 11.0% done | ETA(epoch): 116s + Train loss: 0.876908 (1280.6s) ETA: 6396min + Val loss: 0.856211 [t_0.0-0.2=1.0770 t_0.2-0.4=1.0354 t_0.4-0.6=0.8408 t_0.6-0.8=0.7180 t_0.8-1.0=0.6842] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 33/299 (11% done) --- + [Epoch 33][10/66] loss=0.882820 avg=0.859111 VRAM=39.0GiB | 11.1% done | ETA(epoch): 1089s + [Epoch 33][20/66] loss=1.012192 avg=0.884747 VRAM=38.9GiB | 11.1% done | ETA(epoch): 892s + [Epoch 33][30/66] loss=0.908588 avg=0.889216 VRAM=39.0GiB | 11.2% done | ETA(epoch): 697s + [Epoch 33][40/66] loss=0.928793 avg=0.890994 VRAM=38.9GiB | 11.2% done | ETA(epoch): 503s + [Epoch 33][50/66] loss=0.928457 avg=0.887844 VRAM=39.0GiB | 11.3% done | ETA(epoch): 310s + [Epoch 33][60/66] loss=0.812600 avg=0.882553 VRAM=38.9GiB | 11.3% done | ETA(epoch): 116s + Train loss: 0.881675 (1277.9s) ETA: 6364min + Val loss: 0.874228 [t_0.0-0.2=1.0782 t_0.2-0.4=0.9854 t_0.4-0.6=0.8559 t_0.6-0.8=0.7372 t_0.8-1.0=0.6927] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 34/299 (11% done) --- + [Epoch 34][10/66] loss=0.869256 avg=0.828076 VRAM=39.0GiB | 11.4% done | ETA(epoch): 1082s + [Epoch 34][20/66] loss=0.712645 avg=0.841369 VRAM=38.9GiB | 11.4% done | ETA(epoch): 890s + [Epoch 34][30/66] loss=0.881247 avg=0.841612 VRAM=39.0GiB | 11.5% done | ETA(epoch): 697s + [Epoch 34][40/66] loss=0.816813 avg=0.848897 VRAM=38.9GiB | 11.5% done | ETA(epoch): 503s + [Epoch 34][50/66] loss=0.778003 avg=0.843854 VRAM=39.0GiB | 11.6% done | ETA(epoch): 310s + [Epoch 34][60/66] loss=0.864318 avg=0.845267 VRAM=38.9GiB | 11.6% done | ETA(epoch): 116s + Train loss: 0.847223 (1280.5s) ETA: 6332min + Val loss: 0.887742 [t_0.0-0.2=1.0801 t_0.2-0.4=1.0474 t_0.4-0.6=0.8413 t_0.6-0.8=0.7302 t_0.8-1.0=0.6983] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0034 +[MEM @ epoch 34 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 35/299 (12% done) --- + [Epoch 35][10/66] loss=0.783952 avg=0.843155 VRAM=39.0GiB | 11.7% done | ETA(epoch): 1086s + [Epoch 35][20/66] loss=0.868810 avg=0.856021 VRAM=38.9GiB | 11.8% done | ETA(epoch): 893s + [Epoch 35][30/66] loss=0.805310 avg=0.853535 VRAM=39.0GiB | 11.8% done | ETA(epoch): 699s + [Epoch 35][40/66] loss=0.862571 avg=0.855647 VRAM=38.9GiB | 11.9% done | ETA(epoch): 505s + [Epoch 35][50/66] loss=0.831416 avg=0.857309 VRAM=39.0GiB | 11.9% done | ETA(epoch): 312s + [Epoch 35][60/66] loss=0.928039 avg=0.861717 VRAM=38.9GiB | 12.0% done | ETA(epoch): 117s + Train loss: 0.864182 (1285.2s) ETA: 6302min + Val loss: 0.867428 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9991 t_0.4-0.6=0.8338 t_0.6-0.8=0.7219 t_0.8-1.0=0.6814] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 36/299 (12% done) --- + [Epoch 36][10/66] loss=0.883509 avg=0.922789 VRAM=39.0GiB | 12.1% done | ETA(epoch): 1083s + [Epoch 36][20/66] loss=1.035481 avg=0.930878 VRAM=38.9GiB | 12.1% done | ETA(epoch): 891s + [Epoch 36][30/66] loss=0.927472 avg=0.900271 VRAM=39.0GiB | 12.2% done | ETA(epoch): 698s + [Epoch 36][40/66] loss=0.855651 avg=0.894704 VRAM=38.9GiB | 12.2% done | ETA(epoch): 506s + [Epoch 36][50/66] loss=1.014699 avg=0.885312 VRAM=39.0GiB | 12.3% done | ETA(epoch): 311s + [Epoch 36][60/66] loss=0.974908 avg=0.880543 VRAM=38.9GiB | 12.3% done | ETA(epoch): 117s + Train loss: 0.883056 (1281.5s) ETA: 6272min + Val loss: 0.858711 [t_0.0-0.2=1.0830 t_0.2-0.4=1.0047 t_0.4-0.6=0.8489 t_0.6-0.8=0.7213 t_0.8-1.0=0.6902] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 37/299 (12% done) --- + [Epoch 37][10/66] loss=0.888904 avg=0.842520 VRAM=39.0GiB | 12.4% done | ETA(epoch): 1089s + [Epoch 37][20/66] loss=0.876529 avg=0.843587 VRAM=38.9GiB | 12.4% done | ETA(epoch): 893s + [Epoch 37][30/66] loss=0.859532 avg=0.844998 VRAM=39.0GiB | 12.5% done | ETA(epoch): 699s + [Epoch 37][40/66] loss=0.939788 avg=0.851579 VRAM=38.9GiB | 12.5% done | ETA(epoch): 505s + [Epoch 37][50/66] loss=0.888575 avg=0.848278 VRAM=39.0GiB | 12.6% done | ETA(epoch): 311s + [Epoch 37][60/66] loss=0.815175 avg=0.851215 VRAM=38.9GiB | 12.6% done | ETA(epoch): 117s + Train loss: 0.853965 (1283.4s) ETA: 6242min + Val loss: 0.874666 [t_0.0-0.2=1.0795 t_0.2-0.4=0.9944 t_0.4-0.6=0.8310 t_0.6-0.8=0.7318 t_0.8-1.0=0.6873] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 38/299 (13% done) --- + [Epoch 38][10/66] loss=0.872376 avg=0.850467 VRAM=39.0GiB | 12.7% done | ETA(epoch): 1085s + [Epoch 38][20/66] loss=0.886895 avg=0.868941 VRAM=38.9GiB | 12.8% done | ETA(epoch): 890s + [Epoch 38][30/66] loss=1.043450 avg=0.886163 VRAM=39.0GiB | 12.8% done | ETA(epoch): 697s + [Epoch 38][40/66] loss=0.839288 avg=0.881347 VRAM=38.9GiB | 12.9% done | ETA(epoch): 504s + [Epoch 38][50/66] loss=0.874226 avg=0.880223 VRAM=39.0GiB | 12.9% done | ETA(epoch): 310s + [Epoch 38][60/66] loss=0.767062 avg=0.870621 VRAM=38.9GiB | 13.0% done | ETA(epoch): 116s + Train loss: 0.866661 (1279.3s) ETA: 6212min + Val loss: 0.853515 [t_0.0-0.2=1.0661 t_0.2-0.4=0.9858 t_0.4-0.6=0.8458 t_0.6-0.8=0.7504 t_0.8-1.0=0.6768] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0038 (BEST) + Deleted old checkpoint: checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 39/299 (13% done) --- + [Epoch 39][10/66] loss=0.958250 avg=0.871269 VRAM=39.0GiB | 13.1% done | ETA(epoch): 1097s + [Epoch 39][20/66] loss=0.730777 avg=0.868428 VRAM=38.9GiB | 13.1% done | ETA(epoch): 897s + [Epoch 39][30/66] loss=0.845797 avg=0.863470 VRAM=39.0GiB | 13.2% done | ETA(epoch): 702s + [Epoch 39][40/66] loss=0.891986 avg=0.869749 VRAM=38.9GiB | 13.2% done | ETA(epoch): 506s + [Epoch 39][50/66] loss=0.831349 avg=0.866205 VRAM=39.0GiB | 13.3% done | ETA(epoch): 311s + [Epoch 39][60/66] loss=0.828788 avg=0.868771 VRAM=38.9GiB | 13.3% done | ETA(epoch): 117s + Train loss: 0.868407 (1284.4s) ETA: 6183min + Val loss: 0.852961 [t_0.0-0.2=1.0569 t_0.2-0.4=0.9943 t_0.4-0.6=0.8652 t_0.6-0.8=0.7195 t_0.8-1.0=0.7063] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0039 (BEST) + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 40/299 (13% done) --- + [Epoch 40][10/66] loss=0.773730 avg=0.831726 VRAM=39.0GiB | 13.4% done | ETA(epoch): 1083s + [Epoch 40][20/66] loss=0.919942 avg=0.846203 VRAM=38.9GiB | 13.4% done | ETA(epoch): 889s + [Epoch 40][30/66] loss=0.874544 avg=0.838856 VRAM=39.0GiB | 13.5% done | ETA(epoch): 696s + [Epoch 40][40/66] loss=0.867760 avg=0.839738 VRAM=38.9GiB | 13.5% done | ETA(epoch): 504s + [Epoch 40][50/66] loss=0.764337 avg=0.841558 VRAM=39.0GiB | 13.6% done | ETA(epoch): 310s + [Epoch 40][60/66] loss=0.952024 avg=0.851118 VRAM=38.9GiB | 13.6% done | ETA(epoch): 116s + Train loss: 0.852553 (1280.3s) ETA: 6154min + Val loss: 0.893075 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0326 t_0.4-0.6=0.8765 t_0.6-0.8=0.7116 t_0.8-1.0=0.6746] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 41/299 (14% done) --- + [Epoch 41][10/66] loss=0.821274 avg=0.872956 VRAM=39.0GiB | 13.7% done | ETA(epoch): 1086s + [Epoch 41][20/66] loss=0.820613 avg=0.858590 VRAM=38.9GiB | 13.8% done | ETA(epoch): 891s + [Epoch 41][30/66] loss=0.820060 avg=0.858293 VRAM=39.0GiB | 13.8% done | ETA(epoch): 697s + [Epoch 41][40/66] loss=0.814773 avg=0.849478 VRAM=38.9GiB | 13.9% done | ETA(epoch): 503s + [Epoch 41][50/66] loss=1.026175 avg=0.865219 VRAM=39.0GiB | 13.9% done | ETA(epoch): 310s + [Epoch 41][60/66] loss=0.749472 avg=0.868318 VRAM=38.9GiB | 14.0% done | ETA(epoch): 116s + Train loss: 0.868273 (1280.7s) ETA: 6125min + Val loss: 0.820931 [t_0.0-0.2=1.0518 t_0.2-0.4=0.9884 t_0.4-0.6=0.8687 t_0.6-0.8=0.7268 t_0.8-1.0=0.6750] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0041 (BEST) + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 42/299 (14% done) --- + [Epoch 42][10/66] loss=0.874263 avg=0.854600 VRAM=39.0GiB | 14.1% done | ETA(epoch): 1087s + [Epoch 42][20/66] loss=0.955650 avg=0.835939 VRAM=38.9GiB | 14.1% done | ETA(epoch): 892s + [Epoch 42][30/66] loss=0.941299 avg=0.855007 VRAM=39.0GiB | 14.2% done | ETA(epoch): 700s + [Epoch 42][40/66] loss=0.939066 avg=0.850437 VRAM=38.9GiB | 14.2% done | ETA(epoch): 506s + [Epoch 42][50/66] loss=0.815524 avg=0.848890 VRAM=39.0GiB | 14.3% done | ETA(epoch): 311s + [Epoch 42][60/66] loss=0.877782 avg=0.851669 VRAM=38.9GiB | 14.3% done | ETA(epoch): 117s + Train loss: 0.855595 (1284.5s) ETA: 6097min + Val loss: 0.839107 [t_0.0-0.2=1.0528 t_0.2-0.4=1.0235 t_0.4-0.6=0.8923 t_0.6-0.8=0.6994 t_0.8-1.0=0.6850] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 43/299 (14% done) --- + [Epoch 43][10/66] loss=0.815672 avg=0.847328 VRAM=39.0GiB | 14.4% done | ETA(epoch): 1087s + [Epoch 43][20/66] loss=0.924553 avg=0.853038 VRAM=38.9GiB | 14.4% done | ETA(epoch): 892s + [Epoch 43][30/66] loss=0.792523 avg=0.850887 VRAM=39.0GiB | 14.5% done | ETA(epoch): 697s + [Epoch 43][40/66] loss=0.867906 avg=0.847406 VRAM=38.9GiB | 14.5% done | ETA(epoch): 503s + [Epoch 43][50/66] loss=0.990019 avg=0.853004 VRAM=39.0GiB | 14.6% done | ETA(epoch): 310s + [Epoch 43][60/66] loss=0.899367 avg=0.854369 VRAM=38.9GiB | 14.6% done | ETA(epoch): 116s + Train loss: 0.857847 (1279.7s) ETA: 6069min + Val loss: 0.859859 [t_0.0-0.2=1.0578 t_0.2-0.4=1.0199 t_0.4-0.6=0.8391 t_0.6-0.8=0.7124 t_0.8-1.0=0.6700] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 44/299 (15% done) --- + [Epoch 44][10/66] loss=0.923930 avg=0.874953 VRAM=39.0GiB | 14.7% done | ETA(epoch): 1083s + [Epoch 44][20/66] loss=0.893179 avg=0.884425 VRAM=38.9GiB | 14.8% done | ETA(epoch): 889s + [Epoch 44][30/66] loss=0.800951 avg=0.882577 VRAM=39.0GiB | 14.8% done | ETA(epoch): 700s + [Epoch 44][40/66] loss=0.813014 avg=0.878978 VRAM=38.9GiB | 14.9% done | ETA(epoch): 505s + [Epoch 44][50/66] loss=0.941232 avg=0.873337 VRAM=39.0GiB | 14.9% done | ETA(epoch): 310s + [Epoch 44][60/66] loss=0.760159 avg=0.866017 VRAM=38.9GiB | 15.0% done | ETA(epoch): 116s + Train loss: 0.864950 (1279.7s) ETA: 6041min + Val loss: 0.838116 [t_0.0-0.2=1.0792 t_0.2-0.4=0.9991 t_0.4-0.6=0.8520 t_0.6-0.8=0.7297 t_0.8-1.0=0.6710] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0044 +[MEM @ epoch 44 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 45/299 (15% done) --- + [Epoch 45][10/66] loss=0.926814 avg=0.839023 VRAM=39.0GiB | 15.1% done | ETA(epoch): 1091s + [Epoch 45][20/66] loss=0.872747 avg=0.840217 VRAM=38.9GiB | 15.1% done | ETA(epoch): 897s + [Epoch 45][30/66] loss=0.874952 avg=0.853494 VRAM=39.0GiB | 15.2% done | ETA(epoch): 701s + [Epoch 45][40/66] loss=1.006262 avg=0.862240 VRAM=38.9GiB | 15.2% done | ETA(epoch): 506s + [Epoch 45][50/66] loss=0.949389 avg=0.873373 VRAM=39.0GiB | 15.3% done | ETA(epoch): 311s + [Epoch 45][60/66] loss=0.880021 avg=0.876293 VRAM=38.9GiB | 15.3% done | ETA(epoch): 117s + Train loss: 0.876525 (1282.0s) ETA: 6013min + Val loss: 0.884144 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9856 t_0.4-0.6=0.8841 t_0.6-0.8=0.7038 t_0.8-1.0=0.6703] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 46/299 (15% done) --- + [Epoch 46][10/66] loss=0.943425 avg=0.851188 VRAM=39.0GiB | 15.4% done | ETA(epoch): 1088s + [Epoch 46][20/66] loss=0.782224 avg=0.845298 VRAM=38.9GiB | 15.4% done | ETA(epoch): 893s + [Epoch 46][30/66] loss=0.828131 avg=0.841947 VRAM=39.0GiB | 15.5% done | ETA(epoch): 698s + [Epoch 46][40/66] loss=0.870677 avg=0.852962 VRAM=38.9GiB | 15.5% done | ETA(epoch): 504s + [Epoch 46][50/66] loss=0.842647 avg=0.853376 VRAM=39.0GiB | 15.6% done | ETA(epoch): 310s + [Epoch 46][60/66] loss=0.836513 avg=0.861257 VRAM=38.9GiB | 15.6% done | ETA(epoch): 116s + Train loss: 0.863865 (1279.5s) ETA: 5985min + Val loss: 0.866454 [t_0.0-0.2=1.0891 t_0.2-0.4=0.9755 t_0.4-0.6=0.8408 t_0.6-0.8=0.7229 t_0.8-1.0=0.6680] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 47/299 (16% done) --- + [Epoch 47][10/66] loss=0.797256 avg=0.820244 VRAM=39.0GiB | 15.7% done | ETA(epoch): 1093s + [Epoch 47][20/66] loss=0.900414 avg=0.850658 VRAM=38.9GiB | 15.8% done | ETA(epoch): 895s + [Epoch 47][30/66] loss=0.766936 avg=0.855138 VRAM=39.0GiB | 15.8% done | ETA(epoch): 700s + [Epoch 47][40/66] loss=0.916897 avg=0.857534 VRAM=38.9GiB | 15.9% done | ETA(epoch): 505s + [Epoch 47][50/66] loss=0.871585 avg=0.859297 VRAM=39.0GiB | 15.9% done | ETA(epoch): 311s + [Epoch 47][60/66] loss=1.002937 avg=0.864062 VRAM=38.9GiB | 16.0% done | ETA(epoch): 117s + Train loss: 0.859212 (1283.8s) ETA: 5958min + Val loss: 0.846302 [t_0.0-0.2=1.0994 t_0.2-0.4=1.0300 t_0.4-0.6=0.8564 t_0.6-0.8=0.7081 t_0.8-1.0=0.6661] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 48/299 (16% done) --- + [Epoch 48][10/66] loss=0.750948 avg=0.838065 VRAM=39.0GiB | 16.1% done | ETA(epoch): 1085s + [Epoch 48][20/66] loss=0.869226 avg=0.842540 VRAM=38.9GiB | 16.1% done | ETA(epoch): 893s + [Epoch 48][30/66] loss=0.918566 avg=0.858450 VRAM=39.0GiB | 16.2% done | ETA(epoch): 699s + [Epoch 48][40/66] loss=0.816389 avg=0.860238 VRAM=38.9GiB | 16.2% done | ETA(epoch): 504s + [Epoch 48][50/66] loss=0.969096 avg=0.867560 VRAM=39.0GiB | 16.3% done | ETA(epoch): 310s + [Epoch 48][60/66] loss=0.819170 avg=0.866748 VRAM=38.9GiB | 16.3% done | ETA(epoch): 117s + Train loss: 0.869773 (1282.4s) ETA: 5931min + Val loss: 0.853354 [t_0.0-0.2=1.0763 t_0.2-0.4=1.0105 t_0.4-0.6=0.8777 t_0.6-0.8=0.7201 t_0.8-1.0=0.6654] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 49/299 (16% done) --- + [Epoch 49][10/66] loss=0.967188 avg=0.870973 VRAM=39.0GiB | 16.4% done | ETA(epoch): 1087s + [Epoch 49][20/66] loss=0.917213 avg=0.880626 VRAM=38.9GiB | 16.4% done | ETA(epoch): 893s + [Epoch 49][30/66] loss=0.759983 avg=0.873329 VRAM=39.0GiB | 16.5% done | ETA(epoch): 699s + [Epoch 49][40/66] loss=0.880091 avg=0.864099 VRAM=38.9GiB | 16.5% done | ETA(epoch): 504s + [Epoch 49][50/66] loss=0.959798 avg=0.868691 VRAM=39.0GiB | 16.6% done | ETA(epoch): 311s + [Epoch 49][60/66] loss=0.855856 avg=0.869257 VRAM=38.9GiB | 16.6% done | ETA(epoch): 117s + Train loss: 0.865071 (1281.9s) ETA: 5904min + Val loss: 0.841276 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9944 t_0.4-0.6=0.8293 t_0.6-0.8=0.7092 t_0.8-1.0=0.6802] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 50/299 (17% done) --- + [Epoch 50][10/66] loss=0.740856 avg=0.854169 VRAM=39.0GiB | 16.7% done | ETA(epoch): 1094s + [Epoch 50][20/66] loss=0.785652 avg=0.854175 VRAM=38.9GiB | 16.8% done | ETA(epoch): 898s + [Epoch 50][30/66] loss=0.753741 avg=0.858458 VRAM=39.0GiB | 16.8% done | ETA(epoch): 701s + [Epoch 50][40/66] loss=0.910494 avg=0.865169 VRAM=38.9GiB | 16.9% done | ETA(epoch): 506s + [Epoch 50][50/66] loss=0.794146 avg=0.860631 VRAM=39.0GiB | 16.9% done | ETA(epoch): 312s + [Epoch 50][60/66] loss=0.708256 avg=0.855765 VRAM=38.9GiB | 17.0% done | ETA(epoch): 117s + Train loss: 0.859916 (1283.4s) ETA: 5878min + Val loss: 0.832583 [t_0.0-0.2=1.0647 t_0.2-0.4=0.9716 t_0.4-0.6=0.8516 t_0.6-0.8=0.7260 t_0.8-1.0=0.6468] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 51/299 (17% done) --- + [Epoch 51][10/66] loss=0.849403 avg=0.885791 VRAM=39.0GiB | 17.1% done | ETA(epoch): 1085s + [Epoch 51][20/66] loss=0.889962 avg=0.887621 VRAM=38.9GiB | 17.1% done | ETA(epoch): 892s + [Epoch 51][30/66] loss=0.775822 avg=0.860166 VRAM=39.0GiB | 17.2% done | ETA(epoch): 703s + [Epoch 51][40/66] loss=1.031416 avg=0.860424 VRAM=38.9GiB | 17.2% done | ETA(epoch): 507s + [Epoch 51][50/66] loss=0.887900 avg=0.864705 VRAM=39.0GiB | 17.3% done | ETA(epoch): 312s + [Epoch 51][60/66] loss=0.907278 avg=0.866555 VRAM=38.9GiB | 17.3% done | ETA(epoch): 117s + Train loss: 0.868520 (1285.8s) ETA: 5851min + Val loss: 0.846887 [t_0.0-0.2=1.0725 t_0.2-0.4=1.0015 t_0.4-0.6=0.8136 t_0.6-0.8=0.7413 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 52/299 (17% done) --- + [Epoch 52][10/66] loss=0.817264 avg=0.865811 VRAM=39.0GiB | 17.4% done | ETA(epoch): 1084s + [Epoch 52][20/66] loss=0.865042 avg=0.845578 VRAM=38.9GiB | 17.4% done | ETA(epoch): 902s + [Epoch 52][30/66] loss=0.846708 avg=0.834220 VRAM=39.0GiB | 17.5% done | ETA(epoch): 703s + [Epoch 52][40/66] loss=0.847427 avg=0.848953 VRAM=38.9GiB | 17.5% done | ETA(epoch): 507s + [Epoch 52][50/66] loss=0.772902 avg=0.845438 VRAM=39.0GiB | 17.6% done | ETA(epoch): 312s + [Epoch 52][60/66] loss=0.909283 avg=0.846860 VRAM=38.9GiB | 17.6% done | ETA(epoch): 117s + Train loss: 0.848124 (1284.0s) ETA: 5825min + Val loss: 0.885292 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9832 t_0.4-0.6=0.8237 t_0.6-0.8=0.7219 t_0.8-1.0=0.6766] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 53/299 (18% done) --- + [Epoch 53][10/66] loss=0.913972 avg=0.869117 VRAM=39.0GiB | 17.7% done | ETA(epoch): 1086s + [Epoch 53][20/66] loss=0.804445 avg=0.873560 VRAM=38.9GiB | 17.8% done | ETA(epoch): 892s + [Epoch 53][30/66] loss=0.851018 avg=0.865286 VRAM=39.0GiB | 17.8% done | ETA(epoch): 698s + [Epoch 53][40/66] loss=0.869756 avg=0.857807 VRAM=38.9GiB | 17.9% done | ETA(epoch): 504s + [Epoch 53][50/66] loss=0.784182 avg=0.854276 VRAM=39.0GiB | 17.9% done | ETA(epoch): 310s + [Epoch 53][60/66] loss=0.776377 avg=0.844275 VRAM=38.9GiB | 18.0% done | ETA(epoch): 116s + Train loss: 0.845126 (1279.3s) ETA: 5798min + Val loss: 0.844155 [t_0.0-0.2=1.0787 t_0.2-0.4=0.9543 t_0.4-0.6=0.8380 t_0.6-0.8=0.7467 t_0.8-1.0=0.6629] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 54/299 (18% done) --- + [Epoch 54][10/66] loss=0.772612 avg=0.827598 VRAM=39.0GiB | 18.1% done | ETA(epoch): 1085s + [Epoch 54][20/66] loss=0.841460 avg=0.852731 VRAM=38.9GiB | 18.1% done | ETA(epoch): 890s + [Epoch 54][30/66] loss=0.864496 avg=0.853917 VRAM=39.0GiB | 18.2% done | ETA(epoch): 696s + [Epoch 54][40/66] loss=0.880951 avg=0.852515 VRAM=38.9GiB | 18.2% done | ETA(epoch): 504s + [Epoch 54][50/66] loss=0.926431 avg=0.857249 VRAM=39.0GiB | 18.3% done | ETA(epoch): 310s + [Epoch 54][60/66] loss=0.745588 avg=0.857241 VRAM=38.9GiB | 18.3% done | ETA(epoch): 116s + Train loss: 0.854854 (1279.5s) ETA: 5772min + Val loss: 0.869858 [t_0.0-0.2=1.0574 t_0.2-0.4=1.0146 t_0.4-0.6=0.8657 t_0.6-0.8=0.7298 t_0.8-1.0=0.6671] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 55/299 (18% done) --- + [Epoch 55][10/66] loss=0.878218 avg=0.891779 VRAM=39.0GiB | 18.4% done | ETA(epoch): 1084s + [Epoch 55][20/66] loss=0.862149 avg=0.876131 VRAM=38.9GiB | 18.4% done | ETA(epoch): 890s + [Epoch 55][30/66] loss=0.946842 avg=0.873812 VRAM=39.0GiB | 18.5% done | ETA(epoch): 697s + [Epoch 55][40/66] loss=0.941665 avg=0.870646 VRAM=38.9GiB | 18.5% done | ETA(epoch): 504s + [Epoch 55][50/66] loss=0.932071 avg=0.871034 VRAM=39.0GiB | 18.6% done | ETA(epoch): 310s + [Epoch 55][60/66] loss=0.814053 avg=0.869509 VRAM=38.9GiB | 18.6% done | ETA(epoch): 116s + Train loss: 0.867973 (1281.5s) ETA: 5746min + Val loss: 0.872919 [t_0.0-0.2=1.0716 t_0.2-0.4=1.0222 t_0.4-0.6=0.8506 t_0.6-0.8=0.7189 t_0.8-1.0=0.6675] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 56/299 (19% done) --- + [Epoch 56][10/66] loss=0.953494 avg=0.861335 VRAM=39.0GiB | 18.7% done | ETA(epoch): 1089s + [Epoch 56][20/66] loss=0.946660 avg=0.858893 VRAM=38.9GiB | 18.8% done | ETA(epoch): 893s + [Epoch 56][30/66] loss=0.842210 avg=0.846956 VRAM=39.0GiB | 18.8% done | ETA(epoch): 700s + [Epoch 56][40/66] loss=0.808852 avg=0.859133 VRAM=38.9GiB | 18.9% done | ETA(epoch): 505s + [Epoch 56][50/66] loss=0.705337 avg=0.859523 VRAM=39.0GiB | 18.9% done | ETA(epoch): 311s + [Epoch 56][60/66] loss=0.983142 avg=0.861903 VRAM=38.9GiB | 19.0% done | ETA(epoch): 117s + Train loss: 0.864372 (1283.7s) ETA: 5720min + Val loss: 0.855747 [t_0.0-0.2=1.0555 t_0.2-0.4=1.0051 t_0.4-0.6=0.8325 t_0.6-0.8=0.7373 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 57/299 (19% done) --- + [Epoch 57][10/66] loss=0.764351 avg=0.826242 VRAM=39.0GiB | 19.1% done | ETA(epoch): 1085s + [Epoch 57][20/66] loss=0.966163 avg=0.832392 VRAM=38.9GiB | 19.1% done | ETA(epoch): 893s + [Epoch 57][30/66] loss=0.680617 avg=0.844265 VRAM=39.0GiB | 19.2% done | ETA(epoch): 699s + [Epoch 57][40/66] loss=0.805490 avg=0.842679 VRAM=38.9GiB | 19.2% done | ETA(epoch): 505s + [Epoch 57][50/66] loss=0.757547 avg=0.839673 VRAM=39.0GiB | 19.3% done | ETA(epoch): 310s + [Epoch 57][60/66] loss=0.817513 avg=0.845655 VRAM=38.9GiB | 19.3% done | ETA(epoch): 116s + Train loss: 0.845797 (1279.6s) ETA: 5694min + Val loss: 0.860581 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9888 t_0.4-0.6=0.8451 t_0.6-0.8=0.7328 t_0.8-1.0=0.6540] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 58/299 (19% done) --- + [Epoch 58][10/66] loss=0.742998 avg=0.867721 VRAM=39.0GiB | 19.4% done | ETA(epoch): 1085s + [Epoch 58][20/66] loss=0.979949 avg=0.876753 VRAM=38.9GiB | 19.4% done | ETA(epoch): 891s + [Epoch 58][30/66] loss=0.927968 avg=0.871748 VRAM=39.0GiB | 19.5% done | ETA(epoch): 698s + [Epoch 58][40/66] loss=0.816515 avg=0.864716 VRAM=38.9GiB | 19.5% done | ETA(epoch): 505s + [Epoch 58][50/66] loss=1.031712 avg=0.869502 VRAM=39.0GiB | 19.6% done | ETA(epoch): 311s + [Epoch 58][60/66] loss=0.808012 avg=0.871439 VRAM=38.9GiB | 19.6% done | ETA(epoch): 116s + Train loss: 0.870583 (1280.6s) ETA: 5668min + Val loss: 0.838125 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9971 t_0.4-0.6=0.8686 t_0.6-0.8=0.7106 t_0.8-1.0=0.6514] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 59/299 (20% done) --- + [Epoch 59][10/66] loss=0.709303 avg=0.878862 VRAM=39.0GiB | 19.7% done | ETA(epoch): 1087s + [Epoch 59][20/66] loss=0.751825 avg=0.861981 VRAM=38.9GiB | 19.8% done | ETA(epoch): 899s + [Epoch 59][30/66] loss=0.961051 avg=0.860134 VRAM=39.0GiB | 19.8% done | ETA(epoch): 701s + [Epoch 59][40/66] loss=0.785067 avg=0.861395 VRAM=38.9GiB | 19.9% done | ETA(epoch): 506s + [Epoch 59][50/66] loss=0.936914 avg=0.858547 VRAM=39.0GiB | 19.9% done | ETA(epoch): 311s + [Epoch 59][60/66] loss=0.894875 avg=0.857416 VRAM=38.9GiB | 20.0% done | ETA(epoch): 117s + Train loss: 0.849826 (1283.4s) ETA: 5642min + Val loss: 0.887610 [t_0.0-0.2=1.0662 t_0.2-0.4=0.9717 t_0.4-0.6=0.8665 t_0.6-0.8=0.7381 t_0.8-1.0=0.6580] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 60/299 (20% done) --- + [MilestoneVis] train_0 step 3960 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3960 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3960 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3960 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3960 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3960 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3960 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3960 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3960 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3960 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 60][10/66] loss=0.939948 avg=0.870907 VRAM=39.0GiB | 20.1% done | ETA(epoch): 6866s + [Epoch 60][20/66] loss=0.933524 avg=0.869139 VRAM=38.9GiB | 20.1% done | ETA(epoch): 3270s + [Epoch 60][30/66] loss=0.904009 avg=0.865098 VRAM=39.0GiB | 20.2% done | ETA(epoch): 1938s + [Epoch 60][40/66] loss=0.811895 avg=0.853053 VRAM=38.9GiB | 20.2% done | ETA(epoch): 1176s + [Epoch 60][50/66] loss=0.933482 avg=0.853184 VRAM=39.0GiB | 20.3% done | ETA(epoch): 641s + [Epoch 60][60/66] loss=0.991851 avg=0.853451 VRAM=38.9GiB | 20.3% done | ETA(epoch): 220s + Train loss: 0.856957 (2313.7s) ETA: 5684min + Val loss: 0.859189 [t_0.0-0.2=1.0619 t_0.2-0.4=0.9900 t_0.4-0.6=0.8237 t_0.6-0.8=0.7257 t_0.8-1.0=0.6576] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 61/299 (20% done) --- + [Epoch 61][10/66] loss=0.852278 avg=0.850391 VRAM=39.0GiB | 20.4% done | ETA(epoch): 1083s + [Epoch 61][20/66] loss=0.846825 avg=0.851108 VRAM=38.9GiB | 20.4% done | ETA(epoch): 890s + [Epoch 61][30/66] loss=0.813761 avg=0.859432 VRAM=39.0GiB | 20.5% done | ETA(epoch): 697s + [Epoch 61][40/66] loss=0.865624 avg=0.857460 VRAM=38.9GiB | 20.5% done | ETA(epoch): 503s + [Epoch 61][50/66] loss=0.882973 avg=0.853322 VRAM=39.0GiB | 20.6% done | ETA(epoch): 310s + [Epoch 61][60/66] loss=0.875039 avg=0.854663 VRAM=38.9GiB | 20.6% done | ETA(epoch): 116s + Train loss: 0.856297 (1278.9s) ETA: 5657min + Val loss: 0.857990 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9927 t_0.4-0.6=0.8365 t_0.6-0.8=0.7195 t_0.8-1.0=0.6458] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 62/299 (21% done) --- + [Epoch 62][10/66] loss=0.958577 avg=0.830218 VRAM=39.0GiB | 20.7% done | ETA(epoch): 1088s + [Epoch 62][20/66] loss=0.822832 avg=0.827483 VRAM=38.9GiB | 20.8% done | ETA(epoch): 893s + [Epoch 62][30/66] loss=0.819908 avg=0.840935 VRAM=39.0GiB | 20.8% done | ETA(epoch): 699s + [Epoch 62][40/66] loss=0.893260 avg=0.848673 VRAM=38.9GiB | 20.9% done | ETA(epoch): 505s + [Epoch 62][50/66] loss=0.746982 avg=0.850477 VRAM=39.0GiB | 20.9% done | ETA(epoch): 311s + [Epoch 62][60/66] loss=0.888438 avg=0.852787 VRAM=38.9GiB | 21.0% done | ETA(epoch): 117s + Train loss: 0.853618 (1282.0s) ETA: 5630min + Val loss: 0.854840 [t_0.0-0.2=1.0720 t_0.2-0.4=1.0120 t_0.4-0.6=0.8463 t_0.6-0.8=0.7004 t_0.8-1.0=0.6562] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 63/299 (21% done) --- + [Epoch 63][10/66] loss=0.904472 avg=0.861828 VRAM=39.0GiB | 21.1% done | ETA(epoch): 1081s + [Epoch 63][20/66] loss=0.835430 avg=0.862016 VRAM=38.9GiB | 21.1% done | ETA(epoch): 889s + [Epoch 63][30/66] loss=0.844161 avg=0.860261 VRAM=39.0GiB | 21.2% done | ETA(epoch): 696s + [Epoch 63][40/66] loss=1.025989 avg=0.857180 VRAM=38.9GiB | 21.2% done | ETA(epoch): 503s + [Epoch 63][50/66] loss=0.851450 avg=0.852525 VRAM=39.0GiB | 21.3% done | ETA(epoch): 310s + [Epoch 63][60/66] loss=0.787642 avg=0.850707 VRAM=38.9GiB | 21.3% done | ETA(epoch): 117s + Train loss: 0.850008 (1286.2s) ETA: 5604min + Val loss: 0.848787 [t_0.0-0.2=1.0628 t_0.2-0.4=1.0030 t_0.4-0.6=0.8305 t_0.6-0.8=0.7233 t_0.8-1.0=0.6483] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 64/299 (21% done) --- + [Epoch 64][10/66] loss=0.843370 avg=0.870755 VRAM=39.0GiB | 21.4% done | ETA(epoch): 1083s + [Epoch 64][20/66] loss=0.986745 avg=0.855518 VRAM=38.9GiB | 21.4% done | ETA(epoch): 889s + [Epoch 64][30/66] loss=0.869374 avg=0.856169 VRAM=39.0GiB | 21.5% done | ETA(epoch): 697s + [Epoch 64][40/66] loss=0.889586 avg=0.850904 VRAM=38.9GiB | 21.5% done | ETA(epoch): 503s + [Epoch 64][50/66] loss=0.806162 avg=0.855460 VRAM=39.0GiB | 21.6% done | ETA(epoch): 311s + [Epoch 64][60/66] loss=0.842947 avg=0.859963 VRAM=38.9GiB | 21.6% done | ETA(epoch): 117s + Train loss: 0.858324 (1282.3s) ETA: 5577min + Val loss: 0.865289 [t_0.0-0.2=1.0835 t_0.2-0.4=0.9720 t_0.4-0.6=0.8676 t_0.6-0.8=0.6913 t_0.8-1.0=0.6635] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 65/299 (22% done) --- + [Epoch 65][10/66] loss=0.960664 avg=0.868711 VRAM=39.0GiB | 21.7% done | ETA(epoch): 1092s + [Epoch 65][20/66] loss=0.908806 avg=0.864872 VRAM=38.9GiB | 21.8% done | ETA(epoch): 895s + [Epoch 65][30/66] loss=0.781712 avg=0.875748 VRAM=39.0GiB | 21.8% done | ETA(epoch): 701s + [Epoch 65][40/66] loss=0.910183 avg=0.863711 VRAM=38.9GiB | 21.9% done | ETA(epoch): 509s + [Epoch 65][50/66] loss=0.842681 avg=0.860285 VRAM=39.0GiB | 21.9% done | ETA(epoch): 313s + [Epoch 65][60/66] loss=0.811727 avg=0.858111 VRAM=38.9GiB | 22.0% done | ETA(epoch): 117s + Train loss: 0.858302 (1286.7s) ETA: 5551min + Val loss: 0.864460 [t_0.0-0.2=1.0701 t_0.2-0.4=0.9851 t_0.4-0.6=0.8562 t_0.6-0.8=0.7186 t_0.8-1.0=0.6597] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 66/299 (22% done) --- + [Epoch 66][10/66] loss=0.808305 avg=0.870231 VRAM=39.0GiB | 22.1% done | ETA(epoch): 1087s + [Epoch 66][20/66] loss=0.730016 avg=0.860239 VRAM=38.9GiB | 22.1% done | ETA(epoch): 894s + [Epoch 66][30/66] loss=0.859928 avg=0.848935 VRAM=39.0GiB | 22.2% done | ETA(epoch): 701s + [Epoch 66][40/66] loss=0.758999 avg=0.858774 VRAM=38.9GiB | 22.2% done | ETA(epoch): 506s + [Epoch 66][50/66] loss=0.866826 avg=0.862804 VRAM=39.0GiB | 22.3% done | ETA(epoch): 311s + [Epoch 66][60/66] loss=0.811808 avg=0.861702 VRAM=38.9GiB | 22.3% done | ETA(epoch): 117s + Train loss: 0.865436 (1282.1s) ETA: 5525min + Val loss: 0.832780 [t_0.0-0.2=1.0927 t_0.2-0.4=1.0107 t_0.4-0.6=0.8275 t_0.6-0.8=0.6973 t_0.8-1.0=0.6541] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 67/299 (22% done) --- + [Epoch 67][10/66] loss=0.845906 avg=0.880432 VRAM=39.0GiB | 22.4% done | ETA(epoch): 1084s + [Epoch 67][20/66] loss=0.973867 avg=0.884124 VRAM=38.9GiB | 22.4% done | ETA(epoch): 891s + [Epoch 67][30/66] loss=0.731295 avg=0.873709 VRAM=39.0GiB | 22.5% done | ETA(epoch): 698s + [Epoch 67][40/66] loss=0.835968 avg=0.873753 VRAM=38.9GiB | 22.5% done | ETA(epoch): 504s + [Epoch 67][50/66] loss=0.919466 avg=0.879175 VRAM=39.0GiB | 22.6% done | ETA(epoch): 310s + [Epoch 67][60/66] loss=0.833891 avg=0.876081 VRAM=38.9GiB | 22.6% done | ETA(epoch): 116s + Train loss: 0.873766 (1278.3s) ETA: 5498min + Val loss: 0.870874 [t_0.0-0.2=1.0673 t_0.2-0.4=0.9903 t_0.4-0.6=0.8669 t_0.6-0.8=0.7183 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 68/299 (23% done) --- + [Epoch 68][10/66] loss=0.888420 avg=0.857413 VRAM=39.0GiB | 22.7% done | ETA(epoch): 1085s + [Epoch 68][20/66] loss=0.930921 avg=0.835698 VRAM=38.9GiB | 22.8% done | ETA(epoch): 891s + [Epoch 68][30/66] loss=0.737766 avg=0.835645 VRAM=39.0GiB | 22.8% done | ETA(epoch): 697s + [Epoch 68][40/66] loss=0.875207 avg=0.843490 VRAM=38.9GiB | 22.9% done | ETA(epoch): 503s + [Epoch 68][50/66] loss=0.945980 avg=0.852857 VRAM=39.0GiB | 22.9% done | ETA(epoch): 310s + [Epoch 68][60/66] loss=0.955676 avg=0.847177 VRAM=38.9GiB | 23.0% done | ETA(epoch): 116s + Train loss: 0.846746 (1277.9s) ETA: 5472min + Val loss: 0.855590 [t_0.0-0.2=1.0917 t_0.2-0.4=1.0234 t_0.4-0.6=0.8217 t_0.6-0.8=0.7012 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 69/299 (23% done) --- + [Epoch 69][10/66] loss=0.942623 avg=0.874859 VRAM=39.0GiB | 23.1% done | ETA(epoch): 1083s + [Epoch 69][20/66] loss=0.787385 avg=0.847330 VRAM=38.9GiB | 23.1% done | ETA(epoch): 890s + [Epoch 69][30/66] loss=0.932160 avg=0.853757 VRAM=39.0GiB | 23.2% done | ETA(epoch): 698s + [Epoch 69][40/66] loss=0.756297 avg=0.849819 VRAM=38.9GiB | 23.2% done | ETA(epoch): 505s + [Epoch 69][50/66] loss=0.960853 avg=0.855134 VRAM=39.0GiB | 23.3% done | ETA(epoch): 311s + [Epoch 69][60/66] loss=0.812937 avg=0.855838 VRAM=38.9GiB | 23.3% done | ETA(epoch): 117s + Train loss: 0.849781 (1284.6s) ETA: 5446min + Val loss: 0.852822 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9899 t_0.4-0.6=0.8401 t_0.6-0.8=0.7209 t_0.8-1.0=0.6407] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 70/299 (23% done) --- + [Epoch 70][10/66] loss=0.724889 avg=0.836682 VRAM=39.0GiB | 23.4% done | ETA(epoch): 1086s + [Epoch 70][20/66] loss=0.704360 avg=0.845148 VRAM=38.9GiB | 23.4% done | ETA(epoch): 895s + [Epoch 70][30/66] loss=0.863061 avg=0.847397 VRAM=39.0GiB | 23.5% done | ETA(epoch): 700s + [Epoch 70][40/66] loss=0.934720 avg=0.853969 VRAM=38.9GiB | 23.5% done | ETA(epoch): 505s + [Epoch 70][50/66] loss=0.920218 avg=0.855332 VRAM=39.0GiB | 23.6% done | ETA(epoch): 311s + [Epoch 70][60/66] loss=1.001510 avg=0.856438 VRAM=38.9GiB | 23.6% done | ETA(epoch): 117s + Train loss: 0.857250 (1282.0s) ETA: 5420min + Val loss: 0.839247 [t_0.0-0.2=1.0747 t_0.2-0.4=0.9479 t_0.4-0.6=0.8517 t_0.6-0.8=0.7142 t_0.8-1.0=0.6445] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 71/299 (24% done) --- + [Epoch 71][10/66] loss=0.854324 avg=0.922605 VRAM=39.0GiB | 23.7% done | ETA(epoch): 1090s + [Epoch 71][20/66] loss=0.971948 avg=0.898018 VRAM=38.9GiB | 23.8% done | ETA(epoch): 894s + [Epoch 71][30/66] loss=0.886345 avg=0.865846 VRAM=39.0GiB | 23.8% done | ETA(epoch): 699s + [Epoch 71][40/66] loss=0.731312 avg=0.866155 VRAM=38.9GiB | 23.9% done | ETA(epoch): 505s + [Epoch 71][50/66] loss=0.912125 avg=0.864080 VRAM=39.0GiB | 23.9% done | ETA(epoch): 310s + [Epoch 71][60/66] loss=0.916971 avg=0.864214 VRAM=38.9GiB | 24.0% done | ETA(epoch): 116s + Train loss: 0.865949 (1280.5s) ETA: 5394min + Val loss: 0.849274 [t_0.0-0.2=1.0576 t_0.2-0.4=1.0149 t_0.4-0.6=0.8147 t_0.6-0.8=0.7013 t_0.8-1.0=0.6536] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 72/299 (24% done) --- + [Epoch 72][10/66] loss=0.828167 avg=0.821934 VRAM=39.0GiB | 24.1% done | ETA(epoch): 1085s + [Epoch 72][20/66] loss=0.870169 avg=0.848678 VRAM=38.9GiB | 24.1% done | ETA(epoch): 892s + [Epoch 72][30/66] loss=0.772364 avg=0.851763 VRAM=39.0GiB | 24.2% done | ETA(epoch): 698s + [Epoch 72][40/66] loss=0.792798 avg=0.850123 VRAM=38.9GiB | 24.2% done | ETA(epoch): 505s + [Epoch 72][50/66] loss=0.927794 avg=0.850858 VRAM=39.0GiB | 24.3% done | ETA(epoch): 311s + [Epoch 72][60/66] loss=0.818729 avg=0.851571 VRAM=38.9GiB | 24.3% done | ETA(epoch): 116s + Train loss: 0.854684 (1281.1s) ETA: 5368min + Val loss: 0.842665 [t_0.0-0.2=1.0717 t_0.2-0.4=0.9647 t_0.4-0.6=0.8527 t_0.6-0.8=0.7130 t_0.8-1.0=0.6647] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 73/299 (24% done) --- + [Epoch 73][10/66] loss=0.831164 avg=0.836046 VRAM=39.0GiB | 24.4% done | ETA(epoch): 1086s + [Epoch 73][20/66] loss=0.937300 avg=0.851011 VRAM=38.9GiB | 24.4% done | ETA(epoch): 892s + [Epoch 73][30/66] loss=0.962974 avg=0.857966 VRAM=39.0GiB | 24.5% done | ETA(epoch): 698s + [Epoch 73][40/66] loss=0.895128 avg=0.861384 VRAM=38.9GiB | 24.5% done | ETA(epoch): 504s + [Epoch 73][50/66] loss=0.947748 avg=0.858117 VRAM=39.0GiB | 24.6% done | ETA(epoch): 310s + [Epoch 73][60/66] loss=0.786524 avg=0.858622 VRAM=38.9GiB | 24.6% done | ETA(epoch): 116s + Train loss: 0.857941 (1279.5s) ETA: 5342min + Val loss: 0.908520 [t_0.0-0.2=1.0602 t_0.2-0.4=1.0183 t_0.4-0.6=0.8478 t_0.6-0.8=0.6986 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 74/299 (25% done) --- + [Epoch 74][10/66] loss=0.816865 avg=0.812550 VRAM=39.0GiB | 24.7% done | ETA(epoch): 1087s + [Epoch 74][20/66] loss=0.769484 avg=0.828713 VRAM=38.9GiB | 24.8% done | ETA(epoch): 901s + [Epoch 74][30/66] loss=0.724481 avg=0.833487 VRAM=39.0GiB | 24.8% done | ETA(epoch): 704s + [Epoch 74][40/66] loss=0.902502 avg=0.840073 VRAM=38.9GiB | 24.9% done | ETA(epoch): 507s + [Epoch 74][50/66] loss=0.788206 avg=0.838871 VRAM=39.0GiB | 24.9% done | ETA(epoch): 312s + [Epoch 74][60/66] loss=0.733179 avg=0.838791 VRAM=38.9GiB | 25.0% done | ETA(epoch): 117s + Train loss: 0.839775 (1284.6s) ETA: 5317min + Val loss: 0.869976 [t_0.0-0.2=1.0770 t_0.2-0.4=0.9663 t_0.4-0.6=0.8539 t_0.6-0.8=0.7297 t_0.8-1.0=0.6416] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 75/299 (25% done) --- + [Epoch 75][10/66] loss=0.988038 avg=0.872856 VRAM=39.0GiB | 25.1% done | ETA(epoch): 1107s + [Epoch 75][20/66] loss=0.853985 avg=0.879243 VRAM=38.9GiB | 25.1% done | ETA(epoch): 902s + [Epoch 75][30/66] loss=0.751835 avg=0.862777 VRAM=39.0GiB | 25.2% done | ETA(epoch): 704s + [Epoch 75][40/66] loss=0.835618 avg=0.855385 VRAM=38.9GiB | 25.2% done | ETA(epoch): 507s + [Epoch 75][50/66] loss=0.756835 avg=0.846240 VRAM=39.0GiB | 25.3% done | ETA(epoch): 312s + [Epoch 75][60/66] loss=0.806575 avg=0.854220 VRAM=38.9GiB | 25.3% done | ETA(epoch): 117s + Train loss: 0.850504 (1285.0s) ETA: 5291min + Val loss: 0.882197 [t_0.0-0.2=1.0851 t_0.2-0.4=0.9766 t_0.4-0.6=0.8473 t_0.6-0.8=0.7155 t_0.8-1.0=0.6479] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 76/299 (25% done) --- + [Epoch 76][10/66] loss=0.853570 avg=0.861322 VRAM=39.0GiB | 25.4% done | ETA(epoch): 1087s + [Epoch 76][20/66] loss=0.902969 avg=0.878990 VRAM=38.9GiB | 25.4% done | ETA(epoch): 893s + [Epoch 76][30/66] loss=0.765421 avg=0.870282 VRAM=39.0GiB | 25.5% done | ETA(epoch): 699s + [Epoch 76][40/66] loss=0.913084 avg=0.865925 VRAM=38.9GiB | 25.5% done | ETA(epoch): 505s + [Epoch 76][50/66] loss=0.819927 avg=0.871535 VRAM=39.0GiB | 25.6% done | ETA(epoch): 310s + [Epoch 76][60/66] loss=0.773706 avg=0.868779 VRAM=38.9GiB | 25.6% done | ETA(epoch): 116s + Train loss: 0.862557 (1280.9s) ETA: 5266min + Val loss: 0.843671 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9958 t_0.4-0.6=0.8548 t_0.6-0.8=0.7350 t_0.8-1.0=0.6449] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 77/299 (26% done) --- + [Epoch 77][10/66] loss=0.779742 avg=0.833633 VRAM=39.0GiB | 25.7% done | ETA(epoch): 1088s + [Epoch 77][20/66] loss=0.821860 avg=0.849494 VRAM=38.9GiB | 25.8% done | ETA(epoch): 896s + [Epoch 77][30/66] loss=0.898228 avg=0.847419 VRAM=39.0GiB | 25.8% done | ETA(epoch): 701s + [Epoch 77][40/66] loss=0.876177 avg=0.851102 VRAM=38.9GiB | 25.9% done | ETA(epoch): 506s + [Epoch 77][50/66] loss=0.799384 avg=0.848380 VRAM=39.0GiB | 25.9% done | ETA(epoch): 311s + [Epoch 77][60/66] loss=0.805204 avg=0.851156 VRAM=38.9GiB | 26.0% done | ETA(epoch): 117s + Train loss: 0.854131 (1282.6s) ETA: 5240min + Val loss: 0.837968 [t_0.0-0.2=1.0534 t_0.2-0.4=0.9882 t_0.4-0.6=0.8418 t_0.6-0.8=0.7246 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 78/299 (26% done) --- + [Epoch 78][10/66] loss=0.877779 avg=0.850326 VRAM=39.0GiB | 26.1% done | ETA(epoch): 1086s + [Epoch 78][20/66] loss=0.854462 avg=0.859656 VRAM=38.9GiB | 26.1% done | ETA(epoch): 894s + [Epoch 78][30/66] loss=0.827369 avg=0.856111 VRAM=39.0GiB | 26.2% done | ETA(epoch): 700s + [Epoch 78][40/66] loss=0.915207 avg=0.856259 VRAM=38.9GiB | 26.2% done | ETA(epoch): 505s + [Epoch 78][50/66] loss=0.862785 avg=0.862254 VRAM=39.0GiB | 26.3% done | ETA(epoch): 311s + [Epoch 78][60/66] loss=0.859671 avg=0.870412 VRAM=38.9GiB | 26.3% done | ETA(epoch): 117s + Train loss: 0.866719 (1282.4s) ETA: 5215min + Val loss: 0.829564 [t_0.0-0.2=1.0503 t_0.2-0.4=0.9892 t_0.4-0.6=0.8228 t_0.6-0.8=0.7211 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 79/299 (26% done) --- + [Epoch 79][10/66] loss=0.898513 avg=0.831809 VRAM=39.0GiB | 26.4% done | ETA(epoch): 1088s + [Epoch 79][20/66] loss=0.970885 avg=0.851332 VRAM=38.9GiB | 26.4% done | ETA(epoch): 893s + [Epoch 79][30/66] loss=0.818182 avg=0.858413 VRAM=39.0GiB | 26.5% done | ETA(epoch): 699s + [Epoch 79][40/66] loss=1.012005 avg=0.860511 VRAM=38.9GiB | 26.5% done | ETA(epoch): 510s + [Epoch 79][50/66] loss=0.820073 avg=0.865877 VRAM=39.0GiB | 26.6% done | ETA(epoch): 313s + [Epoch 79][60/66] loss=0.827528 avg=0.864207 VRAM=38.9GiB | 26.6% done | ETA(epoch): 117s + Train loss: 0.858252 (1288.5s) ETA: 5190min + Val loss: 0.824891 [t_0.0-0.2=1.0826 t_0.2-0.4=0.9872 t_0.4-0.6=0.8154 t_0.6-0.8=0.7187 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 80/299 (27% done) --- + [Epoch 80][10/66] loss=0.825321 avg=0.858723 VRAM=39.0GiB | 26.7% done | ETA(epoch): 1087s + [Epoch 80][20/66] loss=0.761778 avg=0.833726 VRAM=38.9GiB | 26.8% done | ETA(epoch): 893s + [Epoch 80][30/66] loss=0.926501 avg=0.840792 VRAM=39.0GiB | 26.8% done | ETA(epoch): 708s + [Epoch 80][40/66] loss=1.040039 avg=0.852664 VRAM=38.9GiB | 26.9% done | ETA(epoch): 510s + [Epoch 80][50/66] loss=0.791164 avg=0.856360 VRAM=39.0GiB | 26.9% done | ETA(epoch): 313s + [Epoch 80][60/66] loss=0.830618 avg=0.858158 VRAM=38.9GiB | 27.0% done | ETA(epoch): 117s + Train loss: 0.862463 (1289.9s) ETA: 5165min + Val loss: 0.867926 [t_0.0-0.2=1.0854 t_0.2-0.4=1.0057 t_0.4-0.6=0.8360 t_0.6-0.8=0.7131 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 81/299 (27% done) --- + [Epoch 81][10/66] loss=1.003597 avg=0.883046 VRAM=39.0GiB | 27.1% done | ETA(epoch): 1089s + [Epoch 81][20/66] loss=0.765024 avg=0.882151 VRAM=38.9GiB | 27.1% done | ETA(epoch): 895s + [Epoch 81][30/66] loss=0.786260 avg=0.867051 VRAM=39.0GiB | 27.2% done | ETA(epoch): 701s + [Epoch 81][40/66] loss=0.837844 avg=0.862383 VRAM=38.9GiB | 27.2% done | ETA(epoch): 506s + [Epoch 81][50/66] loss=0.870454 avg=0.862620 VRAM=39.0GiB | 27.3% done | ETA(epoch): 312s + [Epoch 81][60/66] loss=0.868125 avg=0.863738 VRAM=38.9GiB | 27.3% done | ETA(epoch): 117s + Train loss: 0.864356 (1286.5s) ETA: 5140min + Val loss: 0.855794 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0149 t_0.4-0.6=0.8321 t_0.6-0.8=0.7036 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 82/299 (27% done) --- + [Epoch 82][10/66] loss=1.000680 avg=0.911809 VRAM=39.0GiB | 27.4% done | ETA(epoch): 1091s + [Epoch 82][20/66] loss=0.810055 avg=0.895660 VRAM=38.9GiB | 27.4% done | ETA(epoch): 896s + [Epoch 82][30/66] loss=0.783362 avg=0.869367 VRAM=39.0GiB | 27.5% done | ETA(epoch): 702s + [Epoch 82][40/66] loss=0.951540 avg=0.866629 VRAM=38.9GiB | 27.5% done | ETA(epoch): 507s + [Epoch 82][50/66] loss=0.749254 avg=0.861378 VRAM=39.0GiB | 27.6% done | ETA(epoch): 312s + [Epoch 82][60/66] loss=0.967745 avg=0.861735 VRAM=38.9GiB | 27.6% done | ETA(epoch): 117s + Train loss: 0.859483 (1285.3s) ETA: 5115min + Val loss: 0.900295 [t_0.0-0.2=1.0792 t_0.2-0.4=0.9925 t_0.4-0.6=0.8290 t_0.6-0.8=0.7159 t_0.8-1.0=0.6452] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 83/299 (28% done) --- + [Epoch 83][10/66] loss=0.858915 avg=0.865782 VRAM=39.0GiB | 27.7% done | ETA(epoch): 1088s + [Epoch 83][20/66] loss=0.732223 avg=0.871755 VRAM=38.9GiB | 27.8% done | ETA(epoch): 894s + [Epoch 83][30/66] loss=0.866031 avg=0.866345 VRAM=39.0GiB | 27.8% done | ETA(epoch): 700s + [Epoch 83][40/66] loss=0.872256 avg=0.872033 VRAM=38.9GiB | 27.9% done | ETA(epoch): 506s + [Epoch 83][50/66] loss=0.843704 avg=0.870682 VRAM=39.0GiB | 27.9% done | ETA(epoch): 311s + [Epoch 83][60/66] loss=0.811008 avg=0.872641 VRAM=38.9GiB | 28.0% done | ETA(epoch): 117s + Train loss: 0.871580 (1284.0s) ETA: 5090min + Val loss: 0.847467 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9979 t_0.4-0.6=0.8417 t_0.6-0.8=0.7137 t_0.8-1.0=0.6253] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 84/299 (28% done) --- + [Epoch 84][10/66] loss=0.957634 avg=0.829192 VRAM=39.0GiB | 28.1% done | ETA(epoch): 1089s + [Epoch 84][20/66] loss=0.873738 avg=0.843825 VRAM=38.9GiB | 28.1% done | ETA(epoch): 894s + [Epoch 84][30/66] loss=1.004360 avg=0.847736 VRAM=39.0GiB | 28.2% done | ETA(epoch): 700s + [Epoch 84][40/66] loss=0.761788 avg=0.857153 VRAM=38.9GiB | 28.2% done | ETA(epoch): 506s + [Epoch 84][50/66] loss=0.837020 avg=0.856808 VRAM=39.0GiB | 28.3% done | ETA(epoch): 311s + [Epoch 84][60/66] loss=0.887954 avg=0.855671 VRAM=38.9GiB | 28.3% done | ETA(epoch): 117s + Train loss: 0.857327 (1286.4s) ETA: 5065min + Val loss: 0.842383 [t_0.0-0.2=1.0571 t_0.2-0.4=1.0090 t_0.4-0.6=0.7967 t_0.6-0.8=0.7200 t_0.8-1.0=0.6464] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 85/299 (28% done) --- + [Epoch 85][10/66] loss=0.896496 avg=0.845053 VRAM=39.0GiB | 28.4% done | ETA(epoch): 1087s + [Epoch 85][20/66] loss=0.856171 avg=0.836188 VRAM=38.9GiB | 28.4% done | ETA(epoch): 893s + [Epoch 85][30/66] loss=0.830817 avg=0.842845 VRAM=39.0GiB | 28.5% done | ETA(epoch): 699s + [Epoch 85][40/66] loss=0.986458 avg=0.845162 VRAM=38.9GiB | 28.5% done | ETA(epoch): 505s + [Epoch 85][50/66] loss=0.774013 avg=0.847126 VRAM=39.0GiB | 28.6% done | ETA(epoch): 312s + [Epoch 85][60/66] loss=0.812439 avg=0.843773 VRAM=38.9GiB | 28.6% done | ETA(epoch): 117s + Train loss: 0.843913 (1286.0s) ETA: 5040min + Val loss: 0.844268 [t_0.0-0.2=1.0677 t_0.2-0.4=0.9670 t_0.4-0.6=0.8714 t_0.6-0.8=0.7169 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 86/299 (29% done) --- + [Epoch 86][10/66] loss=0.803018 avg=0.869274 VRAM=39.0GiB | 28.7% done | ETA(epoch): 1085s + [Epoch 86][20/66] loss=0.763766 avg=0.849754 VRAM=38.9GiB | 28.8% done | ETA(epoch): 891s + [Epoch 86][30/66] loss=0.863831 avg=0.858037 VRAM=39.0GiB | 28.8% done | ETA(epoch): 697s + [Epoch 86][40/66] loss=0.792880 avg=0.853003 VRAM=38.9GiB | 28.9% done | ETA(epoch): 504s + [Epoch 86][50/66] loss=0.831806 avg=0.849493 VRAM=39.0GiB | 28.9% done | ETA(epoch): 310s + [Epoch 86][60/66] loss=0.933651 avg=0.851008 VRAM=38.9GiB | 29.0% done | ETA(epoch): 116s + Train loss: 0.850164 (1278.1s) ETA: 5015min + Val loss: 0.869097 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0000 t_0.4-0.6=0.8418 t_0.6-0.8=0.7186 t_0.8-1.0=0.6568] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 87/299 (29% done) --- + [Epoch 87][10/66] loss=0.912020 avg=0.887177 VRAM=39.0GiB | 29.1% done | ETA(epoch): 1084s + [Epoch 87][20/66] loss=0.869962 avg=0.893220 VRAM=38.9GiB | 29.1% done | ETA(epoch): 890s + [Epoch 87][30/66] loss=0.858947 avg=0.875044 VRAM=39.0GiB | 29.2% done | ETA(epoch): 697s + [Epoch 87][40/66] loss=0.902465 avg=0.869264 VRAM=38.9GiB | 29.2% done | ETA(epoch): 503s + [Epoch 87][50/66] loss=0.746655 avg=0.858604 VRAM=39.0GiB | 29.3% done | ETA(epoch): 310s + [Epoch 87][60/66] loss=0.921500 avg=0.861714 VRAM=38.9GiB | 29.3% done | ETA(epoch): 116s + Train loss: 0.858348 (1278.6s) ETA: 4990min + Val loss: 0.853334 [t_0.0-0.2=1.0799 t_0.2-0.4=1.0215 t_0.4-0.6=0.8867 t_0.6-0.8=0.7084 t_0.8-1.0=0.6352] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 88/299 (29% done) --- + [Epoch 88][10/66] loss=0.981882 avg=0.850595 VRAM=39.0GiB | 29.4% done | ETA(epoch): 1091s + [Epoch 88][20/66] loss=0.834325 avg=0.850581 VRAM=38.9GiB | 29.4% done | ETA(epoch): 895s + [Epoch 88][30/66] loss=0.732366 avg=0.860766 VRAM=39.0GiB | 29.5% done | ETA(epoch): 700s + [Epoch 88][40/66] loss=1.018326 avg=0.874544 VRAM=38.9GiB | 29.5% done | ETA(epoch): 505s + [Epoch 88][50/66] loss=0.800186 avg=0.869863 VRAM=39.0GiB | 29.6% done | ETA(epoch): 311s + [Epoch 88][60/66] loss=0.902812 avg=0.862432 VRAM=38.9GiB | 29.6% done | ETA(epoch): 116s + Train loss: 0.866921 (1280.2s) ETA: 4965min + Val loss: 0.861208 [t_0.0-0.2=1.0588 t_0.2-0.4=0.9865 t_0.4-0.6=0.8619 t_0.6-0.8=0.7166 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 89/299 (30% done) --- + [Epoch 89][10/66] loss=0.898059 avg=0.871090 VRAM=39.0GiB | 29.7% done | ETA(epoch): 1087s + [Epoch 89][20/66] loss=0.948959 avg=0.880100 VRAM=38.9GiB | 29.8% done | ETA(epoch): 892s + [Epoch 89][30/66] loss=0.960445 avg=0.877554 VRAM=39.0GiB | 29.8% done | ETA(epoch): 698s + [Epoch 89][40/66] loss=0.973955 avg=0.884002 VRAM=38.9GiB | 29.9% done | ETA(epoch): 504s + [Epoch 89][50/66] loss=0.650226 avg=0.879789 VRAM=39.0GiB | 29.9% done | ETA(epoch): 310s + [Epoch 89][60/66] loss=0.755090 avg=0.877155 VRAM=38.9GiB | 30.0% done | ETA(epoch): 116s + Train loss: 0.875154 (1278.5s) ETA: 4940min + Val loss: 0.854158 [t_0.0-0.2=1.0828 t_0.2-0.4=0.9906 t_0.4-0.6=0.8457 t_0.6-0.8=0.6902 t_0.8-1.0=0.6433] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 90/299 (30% done) --- + [MilestoneVis] train_0 step 5940 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 5940 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 5940 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 5940 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 5940 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 5940 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 5940 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 5940 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 5940 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 5940 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 90][10/66] loss=0.715061 avg=0.869104 VRAM=39.0GiB | 30.1% done | ETA(epoch): 6864s + [Epoch 90][20/66] loss=0.812046 avg=0.850890 VRAM=38.9GiB | 30.1% done | ETA(epoch): 3266s + [Epoch 90][30/66] loss=0.774095 avg=0.844480 VRAM=39.0GiB | 30.2% done | ETA(epoch): 1937s + [Epoch 90][40/66] loss=0.840554 avg=0.846013 VRAM=38.9GiB | 30.2% done | ETA(epoch): 1175s + [Epoch 90][50/66] loss=0.871915 avg=0.846354 VRAM=39.0GiB | 30.3% done | ETA(epoch): 641s + [Epoch 90][60/66] loss=0.846014 avg=0.850029 VRAM=38.9GiB | 30.3% done | ETA(epoch): 220s + Train loss: 0.847606 (2312.7s) ETA: 4955min + Val loss: 0.846912 [t_0.0-0.2=1.0824 t_0.2-0.4=0.9897 t_0.4-0.6=0.8517 t_0.6-0.8=0.6983 t_0.8-1.0=0.6504] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 91/299 (30% done) --- + [Epoch 91][10/66] loss=0.825861 avg=0.865793 VRAM=39.0GiB | 30.4% done | ETA(epoch): 1083s + [Epoch 91][20/66] loss=0.900172 avg=0.868677 VRAM=38.9GiB | 30.4% done | ETA(epoch): 890s + [Epoch 91][30/66] loss=0.719239 avg=0.870686 VRAM=39.0GiB | 30.5% done | ETA(epoch): 696s + [Epoch 91][40/66] loss=0.908703 avg=0.860922 VRAM=38.9GiB | 30.5% done | ETA(epoch): 503s + [Epoch 91][50/66] loss=0.852521 avg=0.861684 VRAM=39.0GiB | 30.6% done | ETA(epoch): 309s + [Epoch 91][60/66] loss=0.831650 avg=0.858428 VRAM=38.9GiB | 30.6% done | ETA(epoch): 116s + Train loss: 0.860788 (1276.4s) ETA: 4929min + Val loss: 0.855483 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0224 t_0.4-0.6=0.8385 t_0.6-0.8=0.7135 t_0.8-1.0=0.6470] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 92/299 (31% done) --- + [Epoch 92][10/66] loss=0.804562 avg=0.833106 VRAM=39.0GiB | 30.7% done | ETA(epoch): 1087s + [Epoch 92][20/66] loss=0.781993 avg=0.827473 VRAM=38.9GiB | 30.8% done | ETA(epoch): 892s + [Epoch 92][30/66] loss=0.907985 avg=0.836114 VRAM=39.0GiB | 30.8% done | ETA(epoch): 698s + [Epoch 92][40/66] loss=0.857403 avg=0.843317 VRAM=38.9GiB | 30.9% done | ETA(epoch): 504s + [Epoch 92][50/66] loss=0.822592 avg=0.841935 VRAM=39.0GiB | 30.9% done | ETA(epoch): 310s + [Epoch 92][60/66] loss=0.937525 avg=0.843711 VRAM=38.9GiB | 31.0% done | ETA(epoch): 116s + Train loss: 0.840890 (1279.9s) ETA: 4904min + Val loss: 0.856014 [t_0.0-0.2=1.0901 t_0.2-0.4=0.9826 t_0.4-0.6=0.8739 t_0.6-0.8=0.7023 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 93/299 (31% done) --- + [Epoch 93][10/66] loss=0.706273 avg=0.847924 VRAM=39.0GiB | 31.1% done | ETA(epoch): 1083s + [Epoch 93][20/66] loss=0.818866 avg=0.838123 VRAM=38.9GiB | 31.1% done | ETA(epoch): 890s + [Epoch 93][30/66] loss=0.866477 avg=0.855272 VRAM=39.0GiB | 31.2% done | ETA(epoch): 696s + [Epoch 93][40/66] loss=0.745366 avg=0.851842 VRAM=38.9GiB | 31.2% done | ETA(epoch): 503s + [Epoch 93][50/66] loss=0.948060 avg=0.854431 VRAM=39.0GiB | 31.3% done | ETA(epoch): 310s + [Epoch 93][60/66] loss=0.786577 avg=0.860102 VRAM=38.9GiB | 31.3% done | ETA(epoch): 116s + Train loss: 0.859044 (1279.2s) ETA: 4878min + Val loss: 0.835316 [t_0.0-0.2=1.0580 t_0.2-0.4=1.0037 t_0.4-0.6=0.8252 t_0.6-0.8=0.7187 t_0.8-1.0=0.6585] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 94/299 (31% done) --- + [Epoch 94][10/66] loss=0.840068 avg=0.833283 VRAM=39.0GiB | 31.4% done | ETA(epoch): 1083s + [Epoch 94][20/66] loss=0.884324 avg=0.842100 VRAM=38.9GiB | 31.4% done | ETA(epoch): 891s + [Epoch 94][30/66] loss=0.855975 avg=0.853883 VRAM=39.0GiB | 31.5% done | ETA(epoch): 697s + [Epoch 94][40/66] loss=0.783533 avg=0.851825 VRAM=38.9GiB | 31.5% done | ETA(epoch): 503s + [Epoch 94][50/66] loss=0.762293 avg=0.847864 VRAM=39.0GiB | 31.6% done | ETA(epoch): 310s + [Epoch 94][60/66] loss=0.736218 avg=0.847185 VRAM=38.9GiB | 31.6% done | ETA(epoch): 116s + Train loss: 0.844031 (1276.4s) ETA: 4853min + Val loss: 0.866639 [t_0.0-0.2=1.0792 t_0.2-0.4=0.9710 t_0.4-0.6=0.8544 t_0.6-0.8=0.7346 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 95/299 (32% done) --- + [Epoch 95][10/66] loss=0.861056 avg=0.874456 VRAM=39.0GiB | 31.7% done | ETA(epoch): 1082s + [Epoch 95][20/66] loss=0.853745 avg=0.838173 VRAM=38.9GiB | 31.8% done | ETA(epoch): 888s + [Epoch 95][30/66] loss=0.835191 avg=0.857048 VRAM=39.0GiB | 31.8% done | ETA(epoch): 699s + [Epoch 95][40/66] loss=0.844486 avg=0.852768 VRAM=38.9GiB | 31.9% done | ETA(epoch): 504s + [Epoch 95][50/66] loss=0.893911 avg=0.857153 VRAM=39.0GiB | 31.9% done | ETA(epoch): 310s + [Epoch 95][60/66] loss=0.984660 avg=0.861042 VRAM=38.9GiB | 32.0% done | ETA(epoch): 116s + Train loss: 0.857667 (1278.5s) ETA: 4828min + Val loss: 0.799058 [t_0.0-0.2=1.0689 t_0.2-0.4=0.9948 t_0.4-0.6=0.8450 t_0.6-0.8=0.7108 t_0.8-1.0=0.6459] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0095 (BEST) + Deleted old checkpoint: checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 96/299 (32% done) --- + [Epoch 96][10/66] loss=1.024536 avg=0.891111 VRAM=39.0GiB | 32.1% done | ETA(epoch): 1083s + [Epoch 96][20/66] loss=0.902020 avg=0.868510 VRAM=38.9GiB | 32.1% done | ETA(epoch): 893s + [Epoch 96][30/66] loss=0.801781 avg=0.867745 VRAM=39.0GiB | 32.2% done | ETA(epoch): 699s + [Epoch 96][40/66] loss=0.873819 avg=0.871848 VRAM=38.9GiB | 32.2% done | ETA(epoch): 504s + [Epoch 96][50/66] loss=0.992048 avg=0.871175 VRAM=39.0GiB | 32.3% done | ETA(epoch): 310s + [Epoch 96][60/66] loss=0.866811 avg=0.868833 VRAM=38.9GiB | 32.3% done | ETA(epoch): 116s + Train loss: 0.863274 (1279.0s) ETA: 4803min + Val loss: 0.862055 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9921 t_0.4-0.6=0.8660 t_0.6-0.8=0.6919 t_0.8-1.0=0.6504] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 97/299 (32% done) --- + [Epoch 97][10/66] loss=0.799881 avg=0.869760 VRAM=39.0GiB | 32.4% done | ETA(epoch): 1085s + [Epoch 97][20/66] loss=0.793100 avg=0.844682 VRAM=38.9GiB | 32.4% done | ETA(epoch): 891s + [Epoch 97][30/66] loss=0.792978 avg=0.844432 VRAM=39.0GiB | 32.5% done | ETA(epoch): 697s + [Epoch 97][40/66] loss=0.834523 avg=0.847046 VRAM=38.9GiB | 32.5% done | ETA(epoch): 503s + [Epoch 97][50/66] loss=0.792896 avg=0.842837 VRAM=39.0GiB | 32.6% done | ETA(epoch): 310s + [Epoch 97][60/66] loss=0.756326 avg=0.840515 VRAM=38.9GiB | 32.6% done | ETA(epoch): 116s + Train loss: 0.843457 (1276.7s) ETA: 4777min + Val loss: 0.799969 [t_0.0-0.2=1.0551 t_0.2-0.4=0.9824 t_0.4-0.6=0.8319 t_0.6-0.8=0.7229 t_0.8-1.0=0.6576] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 18.2/188.5 GiB (9.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 98/299 (33% done) --- + [Epoch 98][10/66] loss=0.861263 avg=0.892954 VRAM=39.0GiB | 32.7% done | ETA(epoch): 1081s + [Epoch 98][20/66] loss=0.866417 avg=0.855012 VRAM=38.9GiB | 32.8% done | ETA(epoch): 888s + [Epoch 98][30/66] loss=0.706958 avg=0.844515 VRAM=39.0GiB | 32.8% done | ETA(epoch): 695s + [Epoch 98][40/66] loss=0.799259 avg=0.836374 VRAM=38.9GiB | 32.9% done | ETA(epoch): 503s + [Epoch 98][50/66] loss=0.865054 avg=0.835492 VRAM=39.0GiB | 32.9% done | ETA(epoch): 310s + [Epoch 98][60/66] loss=0.829103 avg=0.834855 VRAM=38.9GiB | 33.0% done | ETA(epoch): 117s + Train loss: 0.834172 (1282.1s) ETA: 4752min + Val loss: 0.823369 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9918 t_0.4-0.6=0.8576 t_0.6-0.8=0.7208 t_0.8-1.0=0.6553] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0098 +[MEM @ epoch 98 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 99/299 (33% done) --- + [Epoch 99][10/66] loss=0.822526 avg=0.850948 VRAM=39.0GiB | 33.1% done | ETA(epoch): 1081s + [Epoch 99][20/66] loss=0.784479 avg=0.861221 VRAM=38.9GiB | 33.1% done | ETA(epoch): 889s + [Epoch 99][30/66] loss=0.879514 avg=0.867627 VRAM=39.0GiB | 33.2% done | ETA(epoch): 696s + [Epoch 99][40/66] loss=0.727391 avg=0.862099 VRAM=38.9GiB | 33.2% done | ETA(epoch): 503s + [Epoch 99][50/66] loss=0.841062 avg=0.859642 VRAM=39.0GiB | 33.3% done | ETA(epoch): 309s + [Epoch 99][60/66] loss=0.922896 avg=0.860229 VRAM=38.9GiB | 33.3% done | ETA(epoch): 116s + Train loss: 0.858735 (1276.3s) ETA: 4727min + Val loss: 0.830023 [t_0.0-0.2=1.0534 t_0.2-0.4=1.0062 t_0.4-0.6=0.8300 t_0.6-0.8=0.7026 t_0.8-1.0=0.6388] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 100/299 (33% done) --- + [Epoch 100][10/66] loss=0.754427 avg=0.830958 VRAM=39.0GiB | 33.4% done | ETA(epoch): 1083s + [Epoch 100][20/66] loss=0.727458 avg=0.840408 VRAM=38.9GiB | 33.4% done | ETA(epoch): 889s + [Epoch 100][30/66] loss=0.743800 avg=0.838966 VRAM=39.0GiB | 33.5% done | ETA(epoch): 697s + [Epoch 100][40/66] loss=0.865043 avg=0.846676 VRAM=38.9GiB | 33.5% done | ETA(epoch): 503s + [Epoch 100][50/66] loss=0.949054 avg=0.853574 VRAM=39.0GiB | 33.6% done | ETA(epoch): 310s + [Epoch 100][60/66] loss=0.796387 avg=0.850976 VRAM=38.9GiB | 33.6% done | ETA(epoch): 116s + Train loss: 0.851762 (1279.7s) ETA: 4702min + Val loss: 0.820971 [t_0.0-0.2=1.0611 t_0.2-0.4=0.9972 t_0.4-0.6=0.8773 t_0.6-0.8=0.7188 t_0.8-1.0=0.6583] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 101/299 (34% done) --- + [Epoch 101][10/66] loss=0.890612 avg=0.850582 VRAM=39.0GiB | 33.7% done | ETA(epoch): 1083s + [Epoch 101][20/66] loss=0.850035 avg=0.853756 VRAM=38.9GiB | 33.8% done | ETA(epoch): 890s + [Epoch 101][30/66] loss=0.789989 avg=0.849540 VRAM=39.0GiB | 33.8% done | ETA(epoch): 696s + [Epoch 101][40/66] loss=0.895526 avg=0.843594 VRAM=38.9GiB | 33.9% done | ETA(epoch): 504s + [Epoch 101][50/66] loss=0.947384 avg=0.847875 VRAM=39.0GiB | 33.9% done | ETA(epoch): 310s + [Epoch 101][60/66] loss=0.736630 avg=0.840929 VRAM=38.9GiB | 34.0% done | ETA(epoch): 116s + Train loss: 0.844842 (1278.9s) ETA: 4677min + Val loss: 0.832874 [t_0.0-0.2=1.0501 t_0.2-0.4=1.0111 t_0.4-0.6=0.8477 t_0.6-0.8=0.7035 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 102/299 (34% done) --- + [Epoch 102][10/66] loss=0.772696 avg=0.860509 VRAM=39.0GiB | 34.1% done | ETA(epoch): 1082s + [Epoch 102][20/66] loss=0.730798 avg=0.838155 VRAM=38.9GiB | 34.1% done | ETA(epoch): 889s + [Epoch 102][30/66] loss=0.749711 avg=0.842299 VRAM=39.0GiB | 34.2% done | ETA(epoch): 702s + [Epoch 102][40/66] loss=0.846586 avg=0.843495 VRAM=38.9GiB | 34.2% done | ETA(epoch): 506s + [Epoch 102][50/66] loss=0.803657 avg=0.849863 VRAM=39.0GiB | 34.3% done | ETA(epoch): 312s + [Epoch 102][60/66] loss=0.940005 avg=0.852866 VRAM=38.9GiB | 34.3% done | ETA(epoch): 117s + Train loss: 0.860241 (1283.3s) ETA: 4652min + Val loss: 0.856534 [t_0.0-0.2=1.0774 t_0.2-0.4=0.9636 t_0.4-0.6=0.8192 t_0.6-0.8=0.7152 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 103/299 (34% done) --- + [Epoch 103][10/66] loss=0.853691 avg=0.829136 VRAM=39.0GiB | 34.4% done | ETA(epoch): 1081s + [Epoch 103][20/66] loss=0.952423 avg=0.840052 VRAM=38.9GiB | 34.4% done | ETA(epoch): 894s + [Epoch 103][30/66] loss=0.917063 avg=0.855589 VRAM=39.0GiB | 34.5% done | ETA(epoch): 699s + [Epoch 103][40/66] loss=0.790112 avg=0.851170 VRAM=38.9GiB | 34.5% done | ETA(epoch): 504s + [Epoch 103][50/66] loss=0.753522 avg=0.852240 VRAM=39.0GiB | 34.6% done | ETA(epoch): 310s + [Epoch 103][60/66] loss=1.005286 avg=0.854496 VRAM=38.9GiB | 34.6% done | ETA(epoch): 116s + Train loss: 0.849873 (1278.6s) ETA: 4627min + Val loss: 0.860357 [t_0.0-0.2=1.0684 t_0.2-0.4=0.9880 t_0.4-0.6=0.8288 t_0.6-0.8=0.6983 t_0.8-1.0=0.6444] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 104/299 (35% done) --- + [Epoch 104][10/66] loss=0.923446 avg=0.847885 VRAM=39.0GiB | 34.7% done | ETA(epoch): 1093s + [Epoch 104][20/66] loss=0.934042 avg=0.860666 VRAM=38.9GiB | 34.8% done | ETA(epoch): 894s + [Epoch 104][30/66] loss=0.863274 avg=0.865258 VRAM=39.0GiB | 34.8% done | ETA(epoch): 698s + [Epoch 104][40/66] loss=0.863338 avg=0.852497 VRAM=38.9GiB | 34.9% done | ETA(epoch): 503s + [Epoch 104][50/66] loss=0.956326 avg=0.852218 VRAM=39.0GiB | 34.9% done | ETA(epoch): 310s + [Epoch 104][60/66] loss=0.711968 avg=0.853756 VRAM=38.9GiB | 35.0% done | ETA(epoch): 116s + Train loss: 0.855626 (1278.4s) ETA: 4602min + Val loss: 0.859612 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9877 t_0.4-0.6=0.8288 t_0.6-0.8=0.7241 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 105/299 (35% done) --- + [Epoch 105][10/66] loss=0.812636 avg=0.886900 VRAM=39.0GiB | 35.1% done | ETA(epoch): 1084s + [Epoch 105][20/66] loss=0.773259 avg=0.874210 VRAM=38.9GiB | 35.1% done | ETA(epoch): 891s + [Epoch 105][30/66] loss=0.873234 avg=0.871484 VRAM=39.0GiB | 35.2% done | ETA(epoch): 697s + [Epoch 105][40/66] loss=0.938125 avg=0.866999 VRAM=38.9GiB | 35.2% done | ETA(epoch): 504s + [Epoch 105][50/66] loss=0.988277 avg=0.862991 VRAM=39.0GiB | 35.3% done | ETA(epoch): 310s + [Epoch 105][60/66] loss=0.825360 avg=0.861176 VRAM=38.9GiB | 35.3% done | ETA(epoch): 116s + Train loss: 0.862658 (1278.3s) ETA: 4577min + Val loss: 0.862176 [t_0.0-0.2=1.0830 t_0.2-0.4=1.0121 t_0.4-0.6=0.8739 t_0.6-0.8=0.6826 t_0.8-1.0=0.6542] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 106/299 (35% done) --- + [Epoch 106][10/66] loss=0.745710 avg=0.836358 VRAM=39.0GiB | 35.4% done | ETA(epoch): 1084s + [Epoch 106][20/66] loss=0.842983 avg=0.840541 VRAM=38.9GiB | 35.4% done | ETA(epoch): 890s + [Epoch 106][30/66] loss=0.835892 avg=0.838115 VRAM=39.0GiB | 35.5% done | ETA(epoch): 696s + [Epoch 106][40/66] loss=0.857431 avg=0.836395 VRAM=38.9GiB | 35.5% done | ETA(epoch): 503s + [Epoch 106][50/66] loss=0.771740 avg=0.838264 VRAM=39.0GiB | 35.6% done | ETA(epoch): 309s + [Epoch 106][60/66] loss=0.910983 avg=0.837077 VRAM=38.9GiB | 35.6% done | ETA(epoch): 116s + Train loss: 0.838435 (1276.0s) ETA: 4553min + Val loss: 0.876093 [t_0.0-0.2=1.0804 t_0.2-0.4=1.0010 t_0.4-0.6=0.8273 t_0.6-0.8=0.6992 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 107/299 (36% done) --- + [Epoch 107][10/66] loss=0.725822 avg=0.833349 VRAM=39.0GiB | 35.7% done | ETA(epoch): 1083s + [Epoch 107][20/66] loss=0.859583 avg=0.865637 VRAM=38.9GiB | 35.8% done | ETA(epoch): 890s + [Epoch 107][30/66] loss=0.905403 avg=0.863526 VRAM=39.0GiB | 35.8% done | ETA(epoch): 696s + [Epoch 107][40/66] loss=0.755296 avg=0.862688 VRAM=38.9GiB | 35.9% done | ETA(epoch): 503s + [Epoch 107][50/66] loss=0.899236 avg=0.849797 VRAM=39.0GiB | 35.9% done | ETA(epoch): 309s + [Epoch 107][60/66] loss=0.891201 avg=0.853622 VRAM=38.9GiB | 36.0% done | ETA(epoch): 116s + Train loss: 0.851318 (1276.0s) ETA: 4528min + Val loss: 0.868407 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9945 t_0.4-0.6=0.8364 t_0.6-0.8=0.7190 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 108/299 (36% done) --- + [Epoch 108][10/66] loss=0.840572 avg=0.837569 VRAM=39.0GiB | 36.1% done | ETA(epoch): 1084s + [Epoch 108][20/66] loss=0.811499 avg=0.860100 VRAM=38.9GiB | 36.1% done | ETA(epoch): 889s + [Epoch 108][30/66] loss=0.820471 avg=0.853828 VRAM=39.0GiB | 36.2% done | ETA(epoch): 696s + [Epoch 108][40/66] loss=0.851768 avg=0.857276 VRAM=38.9GiB | 36.2% done | ETA(epoch): 503s + [Epoch 108][50/66] loss=0.749031 avg=0.860338 VRAM=39.0GiB | 36.3% done | ETA(epoch): 310s + [Epoch 108][60/66] loss=0.801928 avg=0.856650 VRAM=38.9GiB | 36.3% done | ETA(epoch): 116s + Train loss: 0.862134 (1277.4s) ETA: 4503min + Val loss: 0.858740 [t_0.0-0.2=1.0688 t_0.2-0.4=1.0026 t_0.4-0.6=0.8469 t_0.6-0.8=0.7362 t_0.8-1.0=0.6554] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 109/299 (36% done) --- + [Epoch 109][10/66] loss=0.915140 avg=0.889926 VRAM=39.0GiB | 36.4% done | ETA(epoch): 1085s + [Epoch 109][20/66] loss=0.811782 avg=0.857993 VRAM=38.9GiB | 36.4% done | ETA(epoch): 891s + [Epoch 109][30/66] loss=0.791734 avg=0.848621 VRAM=39.0GiB | 36.5% done | ETA(epoch): 702s + [Epoch 109][40/66] loss=0.832557 avg=0.839405 VRAM=38.9GiB | 36.5% done | ETA(epoch): 505s + [Epoch 109][50/66] loss=0.875672 avg=0.847890 VRAM=39.0GiB | 36.6% done | ETA(epoch): 311s + [Epoch 109][60/66] loss=0.791912 avg=0.838015 VRAM=38.9GiB | 36.6% done | ETA(epoch): 116s + Train loss: 0.837939 (1280.7s) ETA: 4478min + Val loss: 0.892269 [t_0.0-0.2=1.0784 t_0.2-0.4=0.9959 t_0.4-0.6=0.8772 t_0.6-0.8=0.7243 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 110/299 (37% done) --- + [Epoch 110][10/66] loss=0.912709 avg=0.839024 VRAM=39.0GiB | 36.7% done | ETA(epoch): 1082s + [Epoch 110][20/66] loss=0.875065 avg=0.850304 VRAM=38.9GiB | 36.8% done | ETA(epoch): 889s + [Epoch 110][30/66] loss=0.836355 avg=0.859376 VRAM=39.0GiB | 36.8% done | ETA(epoch): 696s + [Epoch 110][40/66] loss=0.763866 avg=0.857387 VRAM=38.9GiB | 36.9% done | ETA(epoch): 503s + [Epoch 110][50/66] loss=0.899678 avg=0.859519 VRAM=39.0GiB | 36.9% done | ETA(epoch): 310s + [Epoch 110][60/66] loss=0.837968 avg=0.853886 VRAM=38.9GiB | 37.0% done | ETA(epoch): 116s + Train loss: 0.853721 (1277.4s) ETA: 4453min + Val loss: 0.857807 [t_0.0-0.2=1.0732 t_0.2-0.4=1.0244 t_0.4-0.6=0.8134 t_0.6-0.8=0.7372 t_0.8-1.0=0.6449] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 111/299 (37% done) --- + [Epoch 111][10/66] loss=0.830207 avg=0.867210 VRAM=39.0GiB | 37.1% done | ETA(epoch): 1079s + [Epoch 111][20/66] loss=0.825004 avg=0.849680 VRAM=38.9GiB | 37.1% done | ETA(epoch): 887s + [Epoch 111][30/66] loss=0.800980 avg=0.851546 VRAM=39.0GiB | 37.2% done | ETA(epoch): 695s + [Epoch 111][40/66] loss=0.966974 avg=0.851097 VRAM=38.9GiB | 37.2% done | ETA(epoch): 502s + [Epoch 111][50/66] loss=0.817604 avg=0.851642 VRAM=39.0GiB | 37.3% done | ETA(epoch): 309s + [Epoch 111][60/66] loss=1.032514 avg=0.859281 VRAM=38.9GiB | 37.3% done | ETA(epoch): 116s + Train loss: 0.859025 (1276.0s) ETA: 4429min + Val loss: 0.871354 [t_0.0-0.2=1.0683 t_0.2-0.4=0.9773 t_0.4-0.6=0.8475 t_0.6-0.8=0.7299 t_0.8-1.0=0.6467] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 112/299 (37% done) --- + [Epoch 112][10/66] loss=0.859166 avg=0.882217 VRAM=39.0GiB | 37.4% done | ETA(epoch): 1080s + [Epoch 112][20/66] loss=0.826716 avg=0.862031 VRAM=38.9GiB | 37.4% done | ETA(epoch): 889s + [Epoch 112][30/66] loss=0.798049 avg=0.846904 VRAM=39.0GiB | 37.5% done | ETA(epoch): 696s + [Epoch 112][40/66] loss=0.883541 avg=0.852042 VRAM=38.9GiB | 37.5% done | ETA(epoch): 503s + [Epoch 112][50/66] loss=0.776350 avg=0.858668 VRAM=39.0GiB | 37.6% done | ETA(epoch): 309s + [Epoch 112][60/66] loss=0.810675 avg=0.860345 VRAM=38.9GiB | 37.6% done | ETA(epoch): 116s + Train loss: 0.858037 (1283.8s) ETA: 4404min + Val loss: 0.859794 [t_0.0-0.2=1.0801 t_0.2-0.4=1.0063 t_0.4-0.6=0.8555 t_0.6-0.8=0.7125 t_0.8-1.0=0.6354] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 113/299 (38% done) --- + [Epoch 113][10/66] loss=0.953899 avg=0.818835 VRAM=39.0GiB | 37.7% done | ETA(epoch): 1082s + [Epoch 113][20/66] loss=0.908580 avg=0.830706 VRAM=38.9GiB | 37.8% done | ETA(epoch): 888s + [Epoch 113][30/66] loss=0.798855 avg=0.835041 VRAM=39.0GiB | 37.8% done | ETA(epoch): 696s + [Epoch 113][40/66] loss=0.862305 avg=0.843571 VRAM=38.9GiB | 37.9% done | ETA(epoch): 502s + [Epoch 113][50/66] loss=0.997241 avg=0.854640 VRAM=39.0GiB | 37.9% done | ETA(epoch): 309s + [Epoch 113][60/66] loss=0.901297 avg=0.850156 VRAM=38.9GiB | 38.0% done | ETA(epoch): 116s + Train loss: 0.852743 (1277.6s) ETA: 4380min + Val loss: 0.834051 [t_0.0-0.2=1.0785 t_0.2-0.4=1.0213 t_0.4-0.6=0.8188 t_0.6-0.8=0.7101 t_0.8-1.0=0.6310] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 114/299 (38% done) --- + [Epoch 114][10/66] loss=0.918886 avg=0.868870 VRAM=39.0GiB | 38.1% done | ETA(epoch): 1083s + [Epoch 114][20/66] loss=0.891437 avg=0.869808 VRAM=38.9GiB | 38.1% done | ETA(epoch): 889s + [Epoch 114][30/66] loss=0.948410 avg=0.861417 VRAM=39.0GiB | 38.2% done | ETA(epoch): 695s + [Epoch 114][40/66] loss=0.862127 avg=0.862499 VRAM=38.9GiB | 38.2% done | ETA(epoch): 502s + [Epoch 114][50/66] loss=0.799892 avg=0.863867 VRAM=39.0GiB | 38.3% done | ETA(epoch): 309s + [Epoch 114][60/66] loss=0.745628 avg=0.855392 VRAM=38.9GiB | 38.3% done | ETA(epoch): 116s + Train loss: 0.847749 (1275.3s) ETA: 4355min + Val loss: 0.849477 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0077 t_0.4-0.6=0.8273 t_0.6-0.8=0.7109 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 115/299 (38% done) --- + [Epoch 115][10/66] loss=0.864831 avg=0.844256 VRAM=39.0GiB | 38.4% done | ETA(epoch): 1082s + [Epoch 115][20/66] loss=0.858175 avg=0.862201 VRAM=38.9GiB | 38.4% done | ETA(epoch): 890s + [Epoch 115][30/66] loss=0.927701 avg=0.860361 VRAM=39.0GiB | 38.5% done | ETA(epoch): 696s + [Epoch 115][40/66] loss=0.837968 avg=0.859022 VRAM=38.9GiB | 38.5% done | ETA(epoch): 506s + [Epoch 115][50/66] loss=0.869806 avg=0.868758 VRAM=39.0GiB | 38.6% done | ETA(epoch): 311s + [Epoch 115][60/66] loss=0.790112 avg=0.867498 VRAM=38.9GiB | 38.6% done | ETA(epoch): 117s + Train loss: 0.862186 (1281.0s) ETA: 4331min + Val loss: 0.887234 [t_0.0-0.2=1.0686 t_0.2-0.4=0.9817 t_0.4-0.6=0.8548 t_0.6-0.8=0.7269 t_0.8-1.0=0.6280] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 116/299 (39% done) --- + [Epoch 116][10/66] loss=0.823686 avg=0.831030 VRAM=39.0GiB | 38.7% done | ETA(epoch): 1082s + [Epoch 116][20/66] loss=0.867637 avg=0.867502 VRAM=38.9GiB | 38.8% done | ETA(epoch): 889s + [Epoch 116][30/66] loss=0.973759 avg=0.860595 VRAM=39.0GiB | 38.8% done | ETA(epoch): 696s + [Epoch 116][40/66] loss=0.883396 avg=0.863659 VRAM=38.9GiB | 38.9% done | ETA(epoch): 503s + [Epoch 116][50/66] loss=0.784563 avg=0.864509 VRAM=39.0GiB | 38.9% done | ETA(epoch): 309s + [Epoch 116][60/66] loss=0.919377 avg=0.861670 VRAM=38.9GiB | 39.0% done | ETA(epoch): 116s + Train loss: 0.855646 (1276.1s) ETA: 4306min + Val loss: 0.841172 [t_0.0-0.2=1.0878 t_0.2-0.4=1.0016 t_0.4-0.6=0.8655 t_0.6-0.8=0.7015 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 117/299 (39% done) --- + [Epoch 117][10/66] loss=0.801577 avg=0.862079 VRAM=39.0GiB | 39.1% done | ETA(epoch): 1083s + [Epoch 117][20/66] loss=0.880031 avg=0.857197 VRAM=38.9GiB | 39.1% done | ETA(epoch): 889s + [Epoch 117][30/66] loss=0.801671 avg=0.855628 VRAM=39.0GiB | 39.2% done | ETA(epoch): 696s + [Epoch 117][40/66] loss=0.852256 avg=0.848117 VRAM=38.9GiB | 39.2% done | ETA(epoch): 502s + [Epoch 117][50/66] loss=0.782341 avg=0.847886 VRAM=39.0GiB | 39.3% done | ETA(epoch): 309s + [Epoch 117][60/66] loss=0.969133 avg=0.847772 VRAM=38.9GiB | 39.3% done | ETA(epoch): 116s + Train loss: 0.845831 (1275.2s) ETA: 4281min + Val loss: 0.846224 [t_0.0-0.2=1.0726 t_0.2-0.4=1.0249 t_0.4-0.6=0.8452 t_0.6-0.8=0.7234 t_0.8-1.0=0.6332] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 118/299 (39% done) --- + [Epoch 118][10/66] loss=0.938661 avg=0.848549 VRAM=39.0GiB | 39.4% done | ETA(epoch): 1087s + [Epoch 118][20/66] loss=0.844012 avg=0.861980 VRAM=38.9GiB | 39.4% done | ETA(epoch): 891s + [Epoch 118][30/66] loss=0.775115 avg=0.858229 VRAM=39.0GiB | 39.5% done | ETA(epoch): 698s + [Epoch 118][40/66] loss=0.899368 avg=0.854557 VRAM=38.9GiB | 39.5% done | ETA(epoch): 504s + [Epoch 118][50/66] loss=0.730065 avg=0.853053 VRAM=39.0GiB | 39.6% done | ETA(epoch): 310s + [Epoch 118][60/66] loss=0.787461 avg=0.845634 VRAM=38.9GiB | 39.6% done | ETA(epoch): 116s + Train loss: 0.846983 (1278.0s) ETA: 4257min + Val loss: 0.827791 [t_0.0-0.2=1.0825 t_0.2-0.4=0.9918 t_0.4-0.6=0.8633 t_0.6-0.8=0.7099 t_0.8-1.0=0.6531] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 119/299 (40% done) --- + [Epoch 119][10/66] loss=0.849841 avg=0.841035 VRAM=39.0GiB | 39.7% done | ETA(epoch): 1095s + [Epoch 119][20/66] loss=0.892330 avg=0.856138 VRAM=38.9GiB | 39.8% done | ETA(epoch): 897s + [Epoch 119][30/66] loss=0.920719 avg=0.853334 VRAM=39.0GiB | 39.8% done | ETA(epoch): 700s + [Epoch 119][40/66] loss=0.882497 avg=0.852385 VRAM=38.9GiB | 39.9% done | ETA(epoch): 505s + [Epoch 119][50/66] loss=0.830517 avg=0.844809 VRAM=39.0GiB | 39.9% done | ETA(epoch): 310s + [Epoch 119][60/66] loss=0.936697 avg=0.845999 VRAM=38.9GiB | 40.0% done | ETA(epoch): 116s + Train loss: 0.847616 (1279.2s) ETA: 4232min + Val loss: 0.879064 [t_0.0-0.2=1.0827 t_0.2-0.4=0.9789 t_0.4-0.6=0.8705 t_0.6-0.8=0.7154 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 120/299 (40% done) --- + [MilestoneVis] train_0 step 7920 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 7920 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 7920 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 7920 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 7920 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 7920 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 7920 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 7920 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 7920 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 7920 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 120][10/66] loss=0.892741 avg=0.850137 VRAM=39.0GiB | 40.1% done | ETA(epoch): 6868s + [Epoch 120][20/66] loss=0.835027 avg=0.828342 VRAM=38.9GiB | 40.1% done | ETA(epoch): 3266s + [Epoch 120][30/66] loss=0.838748 avg=0.847557 VRAM=39.0GiB | 40.2% done | ETA(epoch): 1936s + [Epoch 120][40/66] loss=0.877183 avg=0.843876 VRAM=38.9GiB | 40.2% done | ETA(epoch): 1174s + [Epoch 120][50/66] loss=0.804247 avg=0.847570 VRAM=39.0GiB | 40.3% done | ETA(epoch): 640s + [Epoch 120][60/66] loss=0.781967 avg=0.846175 VRAM=38.9GiB | 40.3% done | ETA(epoch): 219s + Train loss: 0.851026 (2308.6s) ETA: 4233min + Val loss: 0.861851 [t_0.0-0.2=1.0841 t_0.2-0.4=0.9645 t_0.4-0.6=0.8716 t_0.6-0.8=0.7010 t_0.8-1.0=0.6321] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 121/299 (40% done) --- + [Epoch 121][10/66] loss=0.850819 avg=0.845214 VRAM=39.0GiB | 40.4% done | ETA(epoch): 1084s + [Epoch 121][20/66] loss=0.826034 avg=0.845610 VRAM=38.9GiB | 40.4% done | ETA(epoch): 890s + [Epoch 121][30/66] loss=0.895686 avg=0.832412 VRAM=39.0GiB | 40.5% done | ETA(epoch): 696s + [Epoch 121][40/66] loss=0.920796 avg=0.836702 VRAM=38.9GiB | 40.5% done | ETA(epoch): 503s + [Epoch 121][50/66] loss=0.899830 avg=0.836219 VRAM=39.0GiB | 40.6% done | ETA(epoch): 310s + [Epoch 121][60/66] loss=0.911507 avg=0.837384 VRAM=38.9GiB | 40.6% done | ETA(epoch): 116s + Train loss: 0.841125 (1276.3s) ETA: 4209min + Val loss: 0.885705 [t_0.0-0.2=1.0744 t_0.2-0.4=0.9660 t_0.4-0.6=0.9060 t_0.6-0.8=0.7083 t_0.8-1.0=0.6532] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 122/299 (41% done) --- + [Epoch 122][10/66] loss=0.789987 avg=0.840869 VRAM=39.0GiB | 40.7% done | ETA(epoch): 1080s + [Epoch 122][20/66] loss=0.730549 avg=0.858842 VRAM=38.9GiB | 40.8% done | ETA(epoch): 888s + [Epoch 122][30/66] loss=0.823057 avg=0.867137 VRAM=39.0GiB | 40.8% done | ETA(epoch): 695s + [Epoch 122][40/66] loss=0.778898 avg=0.862178 VRAM=38.9GiB | 40.9% done | ETA(epoch): 502s + [Epoch 122][50/66] loss=1.012643 avg=0.864195 VRAM=39.0GiB | 40.9% done | ETA(epoch): 310s + [Epoch 122][60/66] loss=1.032618 avg=0.864655 VRAM=38.9GiB | 41.0% done | ETA(epoch): 116s + Train loss: 0.863113 (1276.8s) ETA: 4184min + Val loss: 0.874145 [t_0.0-0.2=1.0625 t_0.2-0.4=1.0057 t_0.4-0.6=0.8643 t_0.6-0.8=0.7170 t_0.8-1.0=0.6330] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 123/299 (41% done) --- + [Epoch 123][10/66] loss=0.843962 avg=0.822537 VRAM=39.0GiB | 41.1% done | ETA(epoch): 1082s + [Epoch 123][20/66] loss=0.810838 avg=0.835294 VRAM=38.9GiB | 41.1% done | ETA(epoch): 888s + [Epoch 123][30/66] loss=0.898051 avg=0.847198 VRAM=39.0GiB | 41.2% done | ETA(epoch): 695s + [Epoch 123][40/66] loss=0.825149 avg=0.856594 VRAM=38.9GiB | 41.2% done | ETA(epoch): 502s + [Epoch 123][50/66] loss=0.857488 avg=0.852992 VRAM=39.0GiB | 41.3% done | ETA(epoch): 309s + [Epoch 123][60/66] loss=0.902908 avg=0.853510 VRAM=38.9GiB | 41.3% done | ETA(epoch): 116s + Train loss: 0.851370 (1275.2s) ETA: 4159min + Val loss: 0.837544 [t_0.0-0.2=1.0792 t_0.2-0.4=1.0119 t_0.4-0.6=0.8298 t_0.6-0.8=0.7250 t_0.8-1.0=0.6385] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 124/299 (41% done) --- + [Epoch 124][10/66] loss=0.857790 avg=0.828540 VRAM=39.0GiB | 41.4% done | ETA(epoch): 1081s + [Epoch 124][20/66] loss=0.956726 avg=0.843166 VRAM=38.9GiB | 41.4% done | ETA(epoch): 889s + [Epoch 124][30/66] loss=0.919332 avg=0.846413 VRAM=39.0GiB | 41.5% done | ETA(epoch): 696s + [Epoch 124][40/66] loss=0.915845 avg=0.853165 VRAM=38.9GiB | 41.5% done | ETA(epoch): 503s + [Epoch 124][50/66] loss=1.026669 avg=0.856278 VRAM=39.0GiB | 41.6% done | ETA(epoch): 309s + [Epoch 124][60/66] loss=0.772878 avg=0.864933 VRAM=38.9GiB | 41.6% done | ETA(epoch): 116s + Train loss: 0.860445 (1275.8s) ETA: 4134min + Val loss: 0.848847 [t_0.0-0.2=1.0820 t_0.2-0.4=1.0308 t_0.4-0.6=0.8343 t_0.6-0.8=0.6995 t_0.8-1.0=0.6445] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 125/299 (42% done) --- + [Epoch 125][10/66] loss=0.879546 avg=0.887888 VRAM=39.0GiB | 41.7% done | ETA(epoch): 1081s + [Epoch 125][20/66] loss=0.764584 avg=0.878028 VRAM=38.9GiB | 41.8% done | ETA(epoch): 888s + [Epoch 125][30/66] loss=0.794209 avg=0.873589 VRAM=39.0GiB | 41.8% done | ETA(epoch): 698s + [Epoch 125][40/66] loss=0.795065 avg=0.859529 VRAM=38.9GiB | 41.9% done | ETA(epoch): 503s + [Epoch 125][50/66] loss=0.933270 avg=0.860585 VRAM=39.0GiB | 41.9% done | ETA(epoch): 310s + [Epoch 125][60/66] loss=0.861540 avg=0.859956 VRAM=38.9GiB | 42.0% done | ETA(epoch): 116s + Train loss: 0.861623 (1277.1s) ETA: 4110min + Val loss: 0.850563 [t_0.0-0.2=1.0810 t_0.2-0.4=0.9789 t_0.4-0.6=0.8473 t_0.6-0.8=0.7219 t_0.8-1.0=0.6443] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 126/299 (42% done) --- + [Epoch 126][10/66] loss=0.943424 avg=0.860981 VRAM=39.0GiB | 42.1% done | ETA(epoch): 1080s + [Epoch 126][20/66] loss=0.894440 avg=0.862640 VRAM=38.9GiB | 42.1% done | ETA(epoch): 889s + [Epoch 126][30/66] loss=0.941093 avg=0.862851 VRAM=39.0GiB | 42.2% done | ETA(epoch): 696s + [Epoch 126][40/66] loss=0.802417 avg=0.852780 VRAM=38.9GiB | 42.2% done | ETA(epoch): 502s + [Epoch 126][50/66] loss=0.922718 avg=0.845097 VRAM=39.0GiB | 42.3% done | ETA(epoch): 309s + [Epoch 126][60/66] loss=0.799732 avg=0.841049 VRAM=38.9GiB | 42.3% done | ETA(epoch): 116s + Train loss: 0.840821 (1275.1s) ETA: 4085min + Val loss: 0.875629 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9603 t_0.4-0.6=0.8536 t_0.6-0.8=0.7412 t_0.8-1.0=0.6442] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 127/299 (42% done) --- + [Epoch 127][10/66] loss=0.987858 avg=0.857971 VRAM=39.0GiB | 42.4% done | ETA(epoch): 1104s + [Epoch 127][20/66] loss=0.893839 avg=0.824863 VRAM=38.9GiB | 42.4% done | ETA(epoch): 898s + [Epoch 127][30/66] loss=0.912042 avg=0.824131 VRAM=39.0GiB | 42.5% done | ETA(epoch): 700s + [Epoch 127][40/66] loss=0.904980 avg=0.841002 VRAM=38.9GiB | 42.5% done | ETA(epoch): 505s + [Epoch 127][50/66] loss=0.747389 avg=0.839359 VRAM=39.0GiB | 42.6% done | ETA(epoch): 310s + [Epoch 127][60/66] loss=0.920316 avg=0.844801 VRAM=38.9GiB | 42.6% done | ETA(epoch): 116s + Train loss: 0.851536 (1278.2s) ETA: 4060min + Val loss: 0.878115 [t_0.0-0.2=1.0826 t_0.2-0.4=0.9811 t_0.4-0.6=0.8703 t_0.6-0.8=0.7066 t_0.8-1.0=0.6394] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 18.8/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 128/299 (43% done) --- + [Epoch 128][10/66] loss=0.836373 avg=0.864406 VRAM=39.0GiB | 42.7% done | ETA(epoch): 1081s + [Epoch 128][20/66] loss=0.894814 avg=0.882568 VRAM=38.9GiB | 42.8% done | ETA(epoch): 888s + [Epoch 128][30/66] loss=0.930063 avg=0.874922 VRAM=39.0GiB | 42.8% done | ETA(epoch): 695s + [Epoch 128][40/66] loss=0.836803 avg=0.862859 VRAM=38.9GiB | 42.9% done | ETA(epoch): 502s + [Epoch 128][50/66] loss=0.811206 avg=0.866364 VRAM=39.0GiB | 42.9% done | ETA(epoch): 309s + [Epoch 128][60/66] loss=0.962514 avg=0.867620 VRAM=38.9GiB | 43.0% done | ETA(epoch): 116s + Train loss: 0.868515 (1276.1s) ETA: 4036min + Val loss: 0.834205 [t_0.0-0.2=1.0778 t_0.2-0.4=0.9775 t_0.4-0.6=0.8164 t_0.6-0.8=0.7047 t_0.8-1.0=0.6344] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 129/299 (43% done) --- + [Epoch 129][10/66] loss=0.972860 avg=0.844656 VRAM=39.0GiB | 43.1% done | ETA(epoch): 1083s + [Epoch 129][20/66] loss=0.811594 avg=0.858148 VRAM=38.9GiB | 43.1% done | ETA(epoch): 889s + [Epoch 129][30/66] loss=0.901205 avg=0.848248 VRAM=39.0GiB | 43.2% done | ETA(epoch): 696s + [Epoch 129][40/66] loss=0.881118 avg=0.845294 VRAM=38.9GiB | 43.2% done | ETA(epoch): 502s + [Epoch 129][50/66] loss=0.845960 avg=0.841410 VRAM=39.0GiB | 43.3% done | ETA(epoch): 309s + [Epoch 129][60/66] loss=0.749762 avg=0.841887 VRAM=38.9GiB | 43.3% done | ETA(epoch): 116s + Train loss: 0.841197 (1275.2s) ETA: 4011min + Val loss: 0.845425 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9665 t_0.4-0.6=0.8590 t_0.6-0.8=0.7178 t_0.8-1.0=0.6379] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 130/299 (43% done) --- + [Epoch 130][10/66] loss=0.910046 avg=0.876510 VRAM=39.0GiB | 43.4% done | ETA(epoch): 1080s + [Epoch 130][20/66] loss=0.807432 avg=0.890700 VRAM=38.9GiB | 43.4% done | ETA(epoch): 888s + [Epoch 130][30/66] loss=0.900143 avg=0.873920 VRAM=39.0GiB | 43.5% done | ETA(epoch): 695s + [Epoch 130][40/66] loss=0.788369 avg=0.871676 VRAM=38.9GiB | 43.5% done | ETA(epoch): 503s + [Epoch 130][50/66] loss=0.845680 avg=0.874444 VRAM=39.0GiB | 43.6% done | ETA(epoch): 310s + [Epoch 130][60/66] loss=0.895306 avg=0.870425 VRAM=38.9GiB | 43.6% done | ETA(epoch): 116s + Train loss: 0.871608 (1276.1s) ETA: 3987min + Val loss: 0.851555 [t_0.0-0.2=1.0811 t_0.2-0.4=1.0001 t_0.4-0.6=0.8525 t_0.6-0.8=0.7151 t_0.8-1.0=0.6258] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 131/299 (44% done) --- + [Epoch 131][10/66] loss=0.925101 avg=0.855497 VRAM=39.0GiB | 43.7% done | ETA(epoch): 1080s + [Epoch 131][20/66] loss=0.838010 avg=0.866376 VRAM=38.9GiB | 43.8% done | ETA(epoch): 888s + [Epoch 131][30/66] loss=0.725715 avg=0.847642 VRAM=39.0GiB | 43.8% done | ETA(epoch): 695s + [Epoch 131][40/66] loss=0.874900 avg=0.857503 VRAM=38.9GiB | 43.9% done | ETA(epoch): 505s + [Epoch 131][50/66] loss=0.722533 avg=0.854868 VRAM=39.0GiB | 43.9% done | ETA(epoch): 310s + [Epoch 131][60/66] loss=0.785207 avg=0.852946 VRAM=38.9GiB | 44.0% done | ETA(epoch): 116s + Train loss: 0.854881 (1278.6s) ETA: 3962min + Val loss: 0.876397 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9844 t_0.4-0.6=0.8414 t_0.6-0.8=0.7344 t_0.8-1.0=0.6407] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 132/299 (44% done) --- + [Epoch 132][10/66] loss=0.827096 avg=0.863031 VRAM=39.0GiB | 44.1% done | ETA(epoch): 1081s + [Epoch 132][20/66] loss=0.840612 avg=0.846919 VRAM=38.9GiB | 44.1% done | ETA(epoch): 888s + [Epoch 132][30/66] loss=0.812185 avg=0.861889 VRAM=39.0GiB | 44.2% done | ETA(epoch): 695s + [Epoch 132][40/66] loss=0.961752 avg=0.863026 VRAM=38.9GiB | 44.2% done | ETA(epoch): 502s + [Epoch 132][50/66] loss=0.915867 avg=0.867570 VRAM=39.0GiB | 44.3% done | ETA(epoch): 309s + [Epoch 132][60/66] loss=0.727851 avg=0.866200 VRAM=38.9GiB | 44.3% done | ETA(epoch): 116s + Train loss: 0.864663 (1274.9s) ETA: 3938min + Val loss: 0.851434 [t_0.0-0.2=1.0890 t_0.2-0.4=1.0003 t_0.4-0.6=0.8156 t_0.6-0.8=0.7132 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 133/299 (44% done) --- + [Epoch 133][10/66] loss=0.905051 avg=0.881721 VRAM=39.0GiB | 44.4% done | ETA(epoch): 1082s + [Epoch 133][20/66] loss=0.859085 avg=0.853999 VRAM=38.9GiB | 44.4% done | ETA(epoch): 889s + [Epoch 133][30/66] loss=0.679288 avg=0.854952 VRAM=39.0GiB | 44.5% done | ETA(epoch): 696s + [Epoch 133][40/66] loss=0.745954 avg=0.857637 VRAM=38.9GiB | 44.5% done | ETA(epoch): 503s + [Epoch 133][50/66] loss=1.054464 avg=0.857624 VRAM=39.0GiB | 44.6% done | ETA(epoch): 309s + [Epoch 133][60/66] loss=0.902353 avg=0.859812 VRAM=38.9GiB | 44.6% done | ETA(epoch): 116s + Train loss: 0.860194 (1276.5s) ETA: 3913min + Val loss: 0.829310 [t_0.0-0.2=1.0485 t_0.2-0.4=1.0492 t_0.4-0.6=0.8238 t_0.6-0.8=0.7301 t_0.8-1.0=0.6453] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 134/299 (45% done) --- + [Epoch 134][10/66] loss=0.801025 avg=0.844483 VRAM=39.0GiB | 44.7% done | ETA(epoch): 1084s + [Epoch 134][20/66] loss=0.784483 avg=0.824946 VRAM=38.9GiB | 44.8% done | ETA(epoch): 890s + [Epoch 134][30/66] loss=0.976744 avg=0.837766 VRAM=39.0GiB | 44.8% done | ETA(epoch): 697s + [Epoch 134][40/66] loss=0.869942 avg=0.849965 VRAM=38.9GiB | 44.9% done | ETA(epoch): 503s + [Epoch 134][50/66] loss=0.791539 avg=0.857652 VRAM=39.0GiB | 44.9% done | ETA(epoch): 310s + [Epoch 134][60/66] loss=0.854075 avg=0.860983 VRAM=38.9GiB | 45.0% done | ETA(epoch): 116s + Train loss: 0.858525 (1277.3s) ETA: 3889min + Val loss: 0.823404 [t_0.0-0.2=1.0613 t_0.2-0.4=0.9698 t_0.4-0.6=0.8180 t_0.6-0.8=0.7013 t_0.8-1.0=0.6191] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 135/299 (45% done) --- + [Epoch 135][10/66] loss=0.904791 avg=0.823468 VRAM=39.0GiB | 45.1% done | ETA(epoch): 1079s + [Epoch 135][20/66] loss=0.937413 avg=0.840377 VRAM=38.9GiB | 45.1% done | ETA(epoch): 890s + [Epoch 135][30/66] loss=0.736522 avg=0.841657 VRAM=39.0GiB | 45.2% done | ETA(epoch): 697s + [Epoch 135][40/66] loss=0.928927 avg=0.846005 VRAM=38.9GiB | 45.2% done | ETA(epoch): 503s + [Epoch 135][50/66] loss=0.800772 avg=0.853871 VRAM=39.0GiB | 45.3% done | ETA(epoch): 310s + [Epoch 135][60/66] loss=0.840783 avg=0.851039 VRAM=38.9GiB | 45.3% done | ETA(epoch): 116s + Train loss: 0.847560 (1280.1s) ETA: 3865min + Val loss: 0.849554 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9544 t_0.4-0.6=0.8349 t_0.6-0.8=0.7368 t_0.8-1.0=0.6485] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 136/299 (45% done) --- + [Epoch 136][10/66] loss=0.844346 avg=0.900523 VRAM=39.0GiB | 45.4% done | ETA(epoch): 1081s + [Epoch 136][20/66] loss=0.781110 avg=0.892549 VRAM=38.9GiB | 45.4% done | ETA(epoch): 888s + [Epoch 136][30/66] loss=0.944965 avg=0.879537 VRAM=39.0GiB | 45.5% done | ETA(epoch): 695s + [Epoch 136][40/66] loss=0.909294 avg=0.864348 VRAM=38.9GiB | 45.5% done | ETA(epoch): 502s + [Epoch 136][50/66] loss=0.901766 avg=0.863236 VRAM=39.0GiB | 45.6% done | ETA(epoch): 310s + [Epoch 136][60/66] loss=0.704530 avg=0.864460 VRAM=38.9GiB | 45.6% done | ETA(epoch): 116s + Train loss: 0.866006 (1278.2s) ETA: 3840min + Val loss: 0.872642 [t_0.0-0.2=1.0668 t_0.2-0.4=0.9787 t_0.4-0.6=0.8656 t_0.6-0.8=0.7023 t_0.8-1.0=0.6267] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 137/299 (46% done) --- + [Epoch 137][10/66] loss=0.858136 avg=0.853200 VRAM=39.0GiB | 45.7% done | ETA(epoch): 1081s + [Epoch 137][20/66] loss=0.840491 avg=0.856284 VRAM=38.9GiB | 45.8% done | ETA(epoch): 888s + [Epoch 137][30/66] loss=0.917599 avg=0.858192 VRAM=39.0GiB | 45.8% done | ETA(epoch): 696s + [Epoch 137][40/66] loss=0.853458 avg=0.849030 VRAM=38.9GiB | 45.9% done | ETA(epoch): 502s + [Epoch 137][50/66] loss=0.983594 avg=0.850895 VRAM=39.0GiB | 45.9% done | ETA(epoch): 310s + [Epoch 137][60/66] loss=0.918903 avg=0.852221 VRAM=38.9GiB | 46.0% done | ETA(epoch): 116s + Train loss: 0.848497 (1277.4s) ETA: 3816min + Val loss: 0.846566 [t_0.0-0.2=1.0637 t_0.2-0.4=1.0289 t_0.4-0.6=0.8165 t_0.6-0.8=0.7051 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 138/299 (46% done) --- + [Epoch 138][10/66] loss=0.941983 avg=0.814445 VRAM=39.0GiB | 46.1% done | ETA(epoch): 1082s + [Epoch 138][20/66] loss=0.933538 avg=0.829736 VRAM=38.9GiB | 46.1% done | ETA(epoch): 889s + [Epoch 138][30/66] loss=0.848729 avg=0.852749 VRAM=39.0GiB | 46.2% done | ETA(epoch): 697s + [Epoch 138][40/66] loss=0.788238 avg=0.851420 VRAM=38.9GiB | 46.2% done | ETA(epoch): 504s + [Epoch 138][50/66] loss=0.768497 avg=0.853001 VRAM=39.0GiB | 46.3% done | ETA(epoch): 310s + [Epoch 138][60/66] loss=0.844367 avg=0.849430 VRAM=38.9GiB | 46.3% done | ETA(epoch): 116s + Train loss: 0.851164 (1279.9s) ETA: 3792min + Val loss: 0.839789 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9789 t_0.4-0.6=0.8430 t_0.6-0.8=0.7127 t_0.8-1.0=0.6372] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 139/299 (46% done) --- + [Epoch 139][10/66] loss=1.024565 avg=0.889962 VRAM=39.0GiB | 46.4% done | ETA(epoch): 1081s + [Epoch 139][20/66] loss=0.912792 avg=0.877339 VRAM=38.9GiB | 46.4% done | ETA(epoch): 888s + [Epoch 139][30/66] loss=0.750964 avg=0.868006 VRAM=39.0GiB | 46.5% done | ETA(epoch): 696s + [Epoch 139][40/66] loss=0.928407 avg=0.864880 VRAM=38.9GiB | 46.5% done | ETA(epoch): 503s + [Epoch 139][50/66] loss=0.857779 avg=0.861551 VRAM=39.0GiB | 46.6% done | ETA(epoch): 310s + [Epoch 139][60/66] loss=1.020797 avg=0.860970 VRAM=38.9GiB | 46.6% done | ETA(epoch): 116s + Train loss: 0.862944 (1276.5s) ETA: 3767min + Val loss: 0.846050 [t_0.0-0.2=1.0620 t_0.2-0.4=1.0317 t_0.4-0.6=0.8324 t_0.6-0.8=0.7082 t_0.8-1.0=0.6275] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 140/299 (47% done) --- + [Epoch 140][10/66] loss=0.973903 avg=0.874230 VRAM=39.0GiB | 46.7% done | ETA(epoch): 1082s + [Epoch 140][20/66] loss=0.814345 avg=0.871003 VRAM=38.9GiB | 46.8% done | ETA(epoch): 903s + [Epoch 140][30/66] loss=0.829213 avg=0.883540 VRAM=39.0GiB | 46.8% done | ETA(epoch): 703s + [Epoch 140][40/66] loss=0.889353 avg=0.875261 VRAM=38.9GiB | 46.9% done | ETA(epoch): 506s + [Epoch 140][50/66] loss=0.803051 avg=0.864122 VRAM=39.0GiB | 46.9% done | ETA(epoch): 311s + [Epoch 140][60/66] loss=0.883165 avg=0.865381 VRAM=38.9GiB | 47.0% done | ETA(epoch): 117s + Train loss: 0.864079 (1282.9s) ETA: 3743min + Val loss: 0.825662 [t_0.0-0.2=1.0772 t_0.2-0.4=0.9873 t_0.4-0.6=0.8283 t_0.6-0.8=0.7090 t_0.8-1.0=0.6269] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 141/299 (47% done) --- + [Epoch 141][10/66] loss=0.931004 avg=0.872746 VRAM=39.0GiB | 47.1% done | ETA(epoch): 1093s + [Epoch 141][20/66] loss=0.824003 avg=0.869122 VRAM=38.9GiB | 47.1% done | ETA(epoch): 894s + [Epoch 141][30/66] loss=0.948497 avg=0.858912 VRAM=39.0GiB | 47.2% done | ETA(epoch): 698s + [Epoch 141][40/66] loss=0.783060 avg=0.855291 VRAM=38.9GiB | 47.2% done | ETA(epoch): 504s + [Epoch 141][50/66] loss=0.956349 avg=0.854509 VRAM=39.0GiB | 47.3% done | ETA(epoch): 310s + [Epoch 141][60/66] loss=0.905913 avg=0.852148 VRAM=38.9GiB | 47.3% done | ETA(epoch): 116s + Train loss: 0.848484 (1277.5s) ETA: 3719min + Val loss: 0.838706 [t_0.0-0.2=1.0880 t_0.2-0.4=1.0291 t_0.4-0.6=0.8367 t_0.6-0.8=0.7038 t_0.8-1.0=0.6305] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 142/299 (47% done) --- + [Epoch 142][10/66] loss=0.797629 avg=0.852802 VRAM=39.0GiB | 47.4% done | ETA(epoch): 1083s + [Epoch 142][20/66] loss=0.820142 avg=0.842961 VRAM=38.9GiB | 47.4% done | ETA(epoch): 891s + [Epoch 142][30/66] loss=0.766967 avg=0.850868 VRAM=39.0GiB | 47.5% done | ETA(epoch): 698s + [Epoch 142][40/66] loss=0.937212 avg=0.853330 VRAM=38.9GiB | 47.5% done | ETA(epoch): 503s + [Epoch 142][50/66] loss=0.897960 avg=0.853719 VRAM=39.0GiB | 47.6% done | ETA(epoch): 310s + [Epoch 142][60/66] loss=0.910337 avg=0.854484 VRAM=38.9GiB | 47.6% done | ETA(epoch): 116s + Train loss: 0.850785 (1278.3s) ETA: 3695min + Val loss: 0.808973 [t_0.0-0.2=1.0798 t_0.2-0.4=0.9726 t_0.4-0.6=0.8213 t_0.6-0.8=0.7140 t_0.8-1.0=0.6489] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 18.8/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 143/299 (48% done) --- + [Epoch 143][10/66] loss=0.822855 avg=0.892029 VRAM=39.0GiB | 47.7% done | ETA(epoch): 1081s + [Epoch 143][20/66] loss=0.850449 avg=0.864227 VRAM=38.9GiB | 47.8% done | ETA(epoch): 888s + [Epoch 143][30/66] loss=0.851004 avg=0.862599 VRAM=39.0GiB | 47.8% done | ETA(epoch): 695s + [Epoch 143][40/66] loss=0.940467 avg=0.860911 VRAM=38.9GiB | 47.9% done | ETA(epoch): 502s + [Epoch 143][50/66] loss=0.909369 avg=0.858234 VRAM=39.0GiB | 47.9% done | ETA(epoch): 309s + [Epoch 143][60/66] loss=0.823863 avg=0.850594 VRAM=38.9GiB | 48.0% done | ETA(epoch): 116s + Train loss: 0.851357 (1276.5s) ETA: 3670min + Val loss: 0.892374 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9826 t_0.4-0.6=0.8066 t_0.6-0.8=0.7141 t_0.8-1.0=0.6464] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 144/299 (48% done) --- + [Epoch 144][10/66] loss=0.824322 avg=0.846972 VRAM=39.0GiB | 48.1% done | ETA(epoch): 1081s + [Epoch 144][20/66] loss=0.801264 avg=0.844718 VRAM=38.9GiB | 48.1% done | ETA(epoch): 888s + [Epoch 144][30/66] loss=0.869983 avg=0.838691 VRAM=39.0GiB | 48.2% done | ETA(epoch): 695s + [Epoch 144][40/66] loss=0.966033 avg=0.840023 VRAM=38.9GiB | 48.2% done | ETA(epoch): 502s + [Epoch 144][50/66] loss=0.837795 avg=0.843497 VRAM=39.0GiB | 48.3% done | ETA(epoch): 309s + [Epoch 144][60/66] loss=0.935897 avg=0.849059 VRAM=38.9GiB | 48.3% done | ETA(epoch): 116s + Train loss: 0.847485 (1275.9s) ETA: 3646min + Val loss: 0.858483 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9921 t_0.4-0.6=0.8489 t_0.6-0.8=0.6941 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 145/299 (48% done) --- + [Epoch 145][10/66] loss=0.893859 avg=0.859334 VRAM=39.0GiB | 48.4% done | ETA(epoch): 1082s + [Epoch 145][20/66] loss=0.824011 avg=0.856690 VRAM=38.9GiB | 48.4% done | ETA(epoch): 889s + [Epoch 145][30/66] loss=0.854462 avg=0.845742 VRAM=39.0GiB | 48.5% done | ETA(epoch): 697s + [Epoch 145][40/66] loss=0.827210 avg=0.837259 VRAM=38.9GiB | 48.5% done | ETA(epoch): 504s + [Epoch 145][50/66] loss=0.921970 avg=0.843307 VRAM=39.0GiB | 48.6% done | ETA(epoch): 310s + [Epoch 145][60/66] loss=1.024110 avg=0.849990 VRAM=38.9GiB | 48.6% done | ETA(epoch): 116s + Train loss: 0.844896 (1277.4s) ETA: 3622min + Val loss: 0.828916 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9880 t_0.4-0.6=0.8168 t_0.6-0.8=0.7234 t_0.8-1.0=0.6359] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 146/299 (49% done) --- + [Epoch 146][10/66] loss=0.816608 avg=0.830236 VRAM=39.0GiB | 48.7% done | ETA(epoch): 1080s + [Epoch 146][20/66] loss=0.831735 avg=0.834691 VRAM=38.9GiB | 48.8% done | ETA(epoch): 888s + [Epoch 146][30/66] loss=0.920494 avg=0.833243 VRAM=39.0GiB | 48.8% done | ETA(epoch): 695s + [Epoch 146][40/66] loss=0.854375 avg=0.834882 VRAM=38.9GiB | 48.9% done | ETA(epoch): 502s + [Epoch 146][50/66] loss=0.813777 avg=0.839323 VRAM=39.0GiB | 48.9% done | ETA(epoch): 309s + [Epoch 146][60/66] loss=0.749222 avg=0.845915 VRAM=38.9GiB | 49.0% done | ETA(epoch): 116s + Train loss: 0.846460 (1274.4s) ETA: 3598min + Val loss: 0.865019 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9648 t_0.4-0.6=0.8624 t_0.6-0.8=0.7176 t_0.8-1.0=0.6459] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 147/299 (49% done) --- + [Epoch 147][10/66] loss=0.911095 avg=0.832751 VRAM=39.0GiB | 49.1% done | ETA(epoch): 1081s + [Epoch 147][20/66] loss=0.809899 avg=0.833327 VRAM=38.9GiB | 49.1% done | ETA(epoch): 889s + [Epoch 147][30/66] loss=0.968601 avg=0.835420 VRAM=39.0GiB | 49.2% done | ETA(epoch): 696s + [Epoch 147][40/66] loss=0.806305 avg=0.835119 VRAM=38.9GiB | 49.2% done | ETA(epoch): 502s + [Epoch 147][50/66] loss=0.867514 avg=0.834667 VRAM=39.0GiB | 49.3% done | ETA(epoch): 309s + [Epoch 147][60/66] loss=0.817893 avg=0.840580 VRAM=38.9GiB | 49.3% done | ETA(epoch): 116s + Train loss: 0.840269 (1276.0s) ETA: 3573min + Val loss: 0.838206 [t_0.0-0.2=1.0856 t_0.2-0.4=1.0087 t_0.4-0.6=0.8337 t_0.6-0.8=0.6808 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 148/299 (49% done) --- + [Epoch 148][10/66] loss=0.780160 avg=0.817893 VRAM=39.0GiB | 49.4% done | ETA(epoch): 1082s + [Epoch 148][20/66] loss=0.869961 avg=0.848391 VRAM=38.9GiB | 49.4% done | ETA(epoch): 890s + [Epoch 148][30/66] loss=0.873314 avg=0.850913 VRAM=39.0GiB | 49.5% done | ETA(epoch): 697s + [Epoch 148][40/66] loss=0.814132 avg=0.853128 VRAM=38.9GiB | 49.5% done | ETA(epoch): 503s + [Epoch 148][50/66] loss=0.724330 avg=0.855418 VRAM=39.0GiB | 49.6% done | ETA(epoch): 310s + [Epoch 148][60/66] loss=1.015811 avg=0.854927 VRAM=38.9GiB | 49.6% done | ETA(epoch): 116s + Train loss: 0.854589 (1275.9s) ETA: 3549min + Val loss: 0.860277 [t_0.0-0.2=1.0823 t_0.2-0.4=1.0235 t_0.4-0.6=0.8130 t_0.6-0.8=0.7064 t_0.8-1.0=0.6285] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 149/299 (50% done) --- + [Epoch 149][10/66] loss=0.974002 avg=0.896990 VRAM=39.0GiB | 49.7% done | ETA(epoch): 1080s + [Epoch 149][20/66] loss=0.858731 avg=0.873287 VRAM=38.9GiB | 49.8% done | ETA(epoch): 888s + [Epoch 149][30/66] loss=0.842298 avg=0.877815 VRAM=39.0GiB | 49.8% done | ETA(epoch): 695s + [Epoch 149][40/66] loss=0.915595 avg=0.868713 VRAM=38.9GiB | 49.9% done | ETA(epoch): 502s + [Epoch 149][50/66] loss=0.773671 avg=0.874529 VRAM=39.0GiB | 49.9% done | ETA(epoch): 309s + [Epoch 149][60/66] loss=0.918215 avg=0.868233 VRAM=38.9GiB | 50.0% done | ETA(epoch): 116s + Train loss: 0.864321 (1275.2s) ETA: 3525min + Val loss: 0.847880 [t_0.0-0.2=1.0790 t_0.2-0.4=0.9834 t_0.4-0.6=0.8854 t_0.6-0.8=0.7054 t_0.8-1.0=0.6285] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 150/299 (50% done) --- + [MilestoneVis] train_0 step 9900 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9900 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9900 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9900 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9900 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9900 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9900 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9900 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9900 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9900 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/66] loss=0.771617 avg=0.824051 VRAM=39.0GiB | 50.1% done | ETA(epoch): 6884s + [Epoch 150][20/66] loss=0.908813 avg=0.830168 VRAM=38.9GiB | 50.1% done | ETA(epoch): 3272s + [Epoch 150][30/66] loss=0.960095 avg=0.834084 VRAM=39.0GiB | 50.2% done | ETA(epoch): 1939s + [Epoch 150][40/66] loss=0.908837 avg=0.842680 VRAM=38.9GiB | 50.2% done | ETA(epoch): 1176s + [Epoch 150][50/66] loss=0.813159 avg=0.847448 VRAM=39.0GiB | 50.3% done | ETA(epoch): 641s + [Epoch 150][60/66] loss=0.856174 avg=0.843109 VRAM=38.9GiB | 50.3% done | ETA(epoch): 220s + Train loss: 0.838853 (2311.9s) ETA: 3518min + Val loss: 0.843859 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0234 t_0.4-0.6=0.8773 t_0.6-0.8=0.6964 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 151/299 (50% done) --- + [Epoch 151][10/66] loss=0.826943 avg=0.852046 VRAM=39.0GiB | 50.4% done | ETA(epoch): 1080s + [Epoch 151][20/66] loss=0.875794 avg=0.863529 VRAM=38.9GiB | 50.4% done | ETA(epoch): 889s + [Epoch 151][30/66] loss=0.875405 avg=0.859502 VRAM=39.0GiB | 50.5% done | ETA(epoch): 696s + [Epoch 151][40/66] loss=0.740332 avg=0.853036 VRAM=38.9GiB | 50.5% done | ETA(epoch): 503s + [Epoch 151][50/66] loss=0.709655 avg=0.850229 VRAM=39.0GiB | 50.6% done | ETA(epoch): 309s + [Epoch 151][60/66] loss=0.678016 avg=0.843282 VRAM=38.9GiB | 50.6% done | ETA(epoch): 116s + Train loss: 0.849266 (1279.6s) ETA: 3494min + Val loss: 0.834998 [t_0.0-0.2=1.0811 t_0.2-0.4=0.9721 t_0.4-0.6=0.8541 t_0.6-0.8=0.7204 t_0.8-1.0=0.6311] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 152/299 (51% done) --- + [Epoch 152][10/66] loss=0.914558 avg=0.885826 VRAM=39.0GiB | 50.7% done | ETA(epoch): 1079s + [Epoch 152][20/66] loss=0.969225 avg=0.870546 VRAM=38.9GiB | 50.8% done | ETA(epoch): 888s + [Epoch 152][30/66] loss=0.857602 avg=0.860532 VRAM=39.0GiB | 50.8% done | ETA(epoch): 695s + [Epoch 152][40/66] loss=0.896715 avg=0.857796 VRAM=38.9GiB | 50.9% done | ETA(epoch): 502s + [Epoch 152][50/66] loss=0.761045 avg=0.854100 VRAM=39.0GiB | 50.9% done | ETA(epoch): 309s + [Epoch 152][60/66] loss=0.886039 avg=0.856148 VRAM=38.9GiB | 51.0% done | ETA(epoch): 116s + Train loss: 0.855058 (1275.0s) ETA: 3469min + Val loss: 0.868551 [t_0.0-0.2=1.0588 t_0.2-0.4=0.9924 t_0.4-0.6=0.8638 t_0.6-0.8=0.7284 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 19.3/188.5 GiB (10.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 153/299 (51% done) --- + [Epoch 153][10/66] loss=0.938986 avg=0.880377 VRAM=39.0GiB | 51.1% done | ETA(epoch): 1080s + [Epoch 153][20/66] loss=0.781430 avg=0.857435 VRAM=38.9GiB | 51.1% done | ETA(epoch): 888s + [Epoch 153][30/66] loss=0.897474 avg=0.853889 VRAM=39.0GiB | 51.2% done | ETA(epoch): 695s + [Epoch 153][40/66] loss=0.849572 avg=0.863370 VRAM=38.9GiB | 51.2% done | ETA(epoch): 503s + [Epoch 153][50/66] loss=0.779254 avg=0.860133 VRAM=39.0GiB | 51.3% done | ETA(epoch): 310s + [Epoch 153][60/66] loss=0.834403 avg=0.856803 VRAM=38.9GiB | 51.3% done | ETA(epoch): 116s + Train loss: 0.856954 (1276.6s) ETA: 3445min + Val loss: 0.844628 [t_0.0-0.2=1.0805 t_0.2-0.4=0.9712 t_0.4-0.6=0.8104 t_0.6-0.8=0.7456 t_0.8-1.0=0.6190] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 154/299 (51% done) --- + [Epoch 154][10/66] loss=0.724416 avg=0.832087 VRAM=39.0GiB | 51.4% done | ETA(epoch): 1081s + [Epoch 154][20/66] loss=0.832721 avg=0.849933 VRAM=38.9GiB | 51.4% done | ETA(epoch): 888s + [Epoch 154][30/66] loss=0.756192 avg=0.849093 VRAM=39.0GiB | 51.5% done | ETA(epoch): 695s + [Epoch 154][40/66] loss=0.815139 avg=0.859066 VRAM=38.9GiB | 51.5% done | ETA(epoch): 502s + [Epoch 154][50/66] loss=0.932955 avg=0.855333 VRAM=39.0GiB | 51.6% done | ETA(epoch): 309s + [Epoch 154][60/66] loss=0.764792 avg=0.855697 VRAM=38.9GiB | 51.6% done | ETA(epoch): 116s + Train loss: 0.859270 (1274.7s) ETA: 3421min + Val loss: 0.850153 [t_0.0-0.2=1.0677 t_0.2-0.4=0.9610 t_0.4-0.6=0.8303 t_0.6-0.8=0.7276 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 19.3/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 155/299 (52% done) --- + [Epoch 155][10/66] loss=0.763617 avg=0.847402 VRAM=39.0GiB | 51.7% done | ETA(epoch): 1079s + [Epoch 155][20/66] loss=0.867998 avg=0.858130 VRAM=38.9GiB | 51.8% done | ETA(epoch): 887s + [Epoch 155][30/66] loss=0.783644 avg=0.856650 VRAM=39.0GiB | 51.8% done | ETA(epoch): 694s + [Epoch 155][40/66] loss=0.915317 avg=0.853772 VRAM=38.9GiB | 51.9% done | ETA(epoch): 502s + [Epoch 155][50/66] loss=0.895962 avg=0.846276 VRAM=39.0GiB | 51.9% done | ETA(epoch): 309s + [Epoch 155][60/66] loss=0.838973 avg=0.847145 VRAM=38.9GiB | 52.0% done | ETA(epoch): 116s + Train loss: 0.848806 (1276.1s) ETA: 3397min + Val loss: 0.835536 [t_0.0-0.2=1.0544 t_0.2-0.4=0.9942 t_0.4-0.6=0.8754 t_0.6-0.8=0.7181 t_0.8-1.0=0.6253] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 156/299 (52% done) --- + [Epoch 156][10/66] loss=0.887012 avg=0.846052 VRAM=39.0GiB | 52.1% done | ETA(epoch): 1082s + [Epoch 156][20/66] loss=0.911209 avg=0.845525 VRAM=38.9GiB | 52.1% done | ETA(epoch): 889s + [Epoch 156][30/66] loss=0.871162 avg=0.853951 VRAM=39.0GiB | 52.2% done | ETA(epoch): 695s + [Epoch 156][40/66] loss=0.926561 avg=0.855661 VRAM=38.9GiB | 52.2% done | ETA(epoch): 502s + [Epoch 156][50/66] loss=0.945397 avg=0.855548 VRAM=39.0GiB | 52.3% done | ETA(epoch): 309s + [Epoch 156][60/66] loss=0.869881 avg=0.855500 VRAM=38.9GiB | 52.3% done | ETA(epoch): 116s + Train loss: 0.853360 (1274.8s) ETA: 3372min + Val loss: 0.841035 [t_0.0-0.2=1.0795 t_0.2-0.4=0.9888 t_0.4-0.6=0.8269 t_0.6-0.8=0.7377 t_0.8-1.0=0.6375] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 157/299 (52% done) --- + [Epoch 157][10/66] loss=0.930567 avg=0.825514 VRAM=39.0GiB | 52.4% done | ETA(epoch): 1127s + [Epoch 157][20/66] loss=0.881687 avg=0.839781 VRAM=38.9GiB | 52.4% done | ETA(epoch): 909s + [Epoch 157][30/66] loss=0.812184 avg=0.818780 VRAM=39.0GiB | 52.5% done | ETA(epoch): 707s + [Epoch 157][40/66] loss=0.923815 avg=0.830227 VRAM=38.9GiB | 52.5% done | ETA(epoch): 509s + [Epoch 157][50/66] loss=0.868942 avg=0.835589 VRAM=39.0GiB | 52.6% done | ETA(epoch): 313s + [Epoch 157][60/66] loss=0.914437 avg=0.837234 VRAM=38.9GiB | 52.6% done | ETA(epoch): 117s + Train loss: 0.841062 (1288.4s) ETA: 3348min + Val loss: 0.853356 [t_0.0-0.2=1.0630 t_0.2-0.4=1.0030 t_0.4-0.6=0.8677 t_0.6-0.8=0.7332 t_0.8-1.0=0.6091] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 158/299 (53% done) --- + [Epoch 158][10/66] loss=0.742356 avg=0.880906 VRAM=39.0GiB | 52.7% done | ETA(epoch): 1083s + [Epoch 158][20/66] loss=0.807256 avg=0.879461 VRAM=38.9GiB | 52.8% done | ETA(epoch): 889s + [Epoch 158][30/66] loss=0.780048 avg=0.862380 VRAM=39.0GiB | 52.8% done | ETA(epoch): 696s + [Epoch 158][40/66] loss=0.809184 avg=0.848357 VRAM=38.9GiB | 52.9% done | ETA(epoch): 503s + [Epoch 158][50/66] loss=0.904671 avg=0.851317 VRAM=39.0GiB | 52.9% done | ETA(epoch): 310s + [Epoch 158][60/66] loss=0.738748 avg=0.851568 VRAM=38.9GiB | 53.0% done | ETA(epoch): 116s + Train loss: 0.852059 (1277.4s) ETA: 3324min + Val loss: 0.869319 [t_0.0-0.2=1.0583 t_0.2-0.4=1.0081 t_0.4-0.6=0.8265 t_0.6-0.8=0.7182 t_0.8-1.0=0.6357] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 19.5/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 159/299 (53% done) --- + [Epoch 159][10/66] loss=0.869611 avg=0.867354 VRAM=39.0GiB | 53.1% done | ETA(epoch): 1080s + [Epoch 159][20/66] loss=0.788093 avg=0.881085 VRAM=38.9GiB | 53.1% done | ETA(epoch): 889s + [Epoch 159][30/66] loss=0.887939 avg=0.874398 VRAM=39.0GiB | 53.2% done | ETA(epoch): 696s + [Epoch 159][40/66] loss=0.861750 avg=0.868382 VRAM=38.9GiB | 53.2% done | ETA(epoch): 503s + [Epoch 159][50/66] loss=0.882440 avg=0.865599 VRAM=39.0GiB | 53.3% done | ETA(epoch): 310s + [Epoch 159][60/66] loss=1.039539 avg=0.872498 VRAM=38.9GiB | 53.3% done | ETA(epoch): 116s + Train loss: 0.866815 (1278.1s) ETA: 3300min + Val loss: 0.851951 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9874 t_0.4-0.6=0.8661 t_0.6-0.8=0.7110 t_0.8-1.0=0.6324] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 160/299 (53% done) --- + [Epoch 160][10/66] loss=0.816528 avg=0.846009 VRAM=39.0GiB | 53.4% done | ETA(epoch): 1081s + [Epoch 160][20/66] loss=0.760734 avg=0.850516 VRAM=38.9GiB | 53.4% done | ETA(epoch): 889s + [Epoch 160][30/66] loss=0.783501 avg=0.838106 VRAM=39.0GiB | 53.5% done | ETA(epoch): 696s + [Epoch 160][40/66] loss=0.882233 avg=0.836670 VRAM=38.9GiB | 53.5% done | ETA(epoch): 502s + [Epoch 160][50/66] loss=0.868765 avg=0.840507 VRAM=39.0GiB | 53.6% done | ETA(epoch): 309s + [Epoch 160][60/66] loss=1.005332 avg=0.848698 VRAM=38.9GiB | 53.6% done | ETA(epoch): 116s + Train loss: 0.846983 (1275.7s) ETA: 3276min + Val loss: 0.893407 [t_0.0-0.2=1.0870 t_0.2-0.4=1.0087 t_0.4-0.6=0.8534 t_0.6-0.8=0.7382 t_0.8-1.0=0.6273] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 161/299 (54% done) --- + [Epoch 161][10/66] loss=0.869804 avg=0.885133 VRAM=39.0GiB | 53.7% done | ETA(epoch): 1081s + [Epoch 161][20/66] loss=0.805610 avg=0.866968 VRAM=38.9GiB | 53.8% done | ETA(epoch): 889s + [Epoch 161][30/66] loss=0.843725 avg=0.860077 VRAM=39.0GiB | 53.8% done | ETA(epoch): 697s + [Epoch 161][40/66] loss=0.850151 avg=0.860263 VRAM=38.9GiB | 53.9% done | ETA(epoch): 503s + [Epoch 161][50/66] loss=0.704453 avg=0.845668 VRAM=39.0GiB | 53.9% done | ETA(epoch): 309s + [Epoch 161][60/66] loss=0.936031 avg=0.846996 VRAM=38.9GiB | 54.0% done | ETA(epoch): 116s + Train loss: 0.852451 (1276.5s) ETA: 3251min + Val loss: 0.846999 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9900 t_0.4-0.6=0.8393 t_0.6-0.8=0.7214 t_0.8-1.0=0.6372] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 162/299 (54% done) --- + [Epoch 162][10/66] loss=0.860813 avg=0.818862 VRAM=39.0GiB | 54.1% done | ETA(epoch): 1084s + [Epoch 162][20/66] loss=0.861506 avg=0.852801 VRAM=38.9GiB | 54.1% done | ETA(epoch): 900s + [Epoch 162][30/66] loss=0.825152 avg=0.846752 VRAM=39.0GiB | 54.2% done | ETA(epoch): 702s + [Epoch 162][40/66] loss=0.867462 avg=0.855568 VRAM=38.9GiB | 54.2% done | ETA(epoch): 506s + [Epoch 162][50/66] loss=0.847040 avg=0.857625 VRAM=39.0GiB | 54.3% done | ETA(epoch): 311s + [Epoch 162][60/66] loss=0.871890 avg=0.858097 VRAM=38.9GiB | 54.3% done | ETA(epoch): 117s + Train loss: 0.856701 (1281.8s) ETA: 3227min + Val loss: 0.836465 [t_0.0-0.2=1.0859 t_0.2-0.4=0.9369 t_0.4-0.6=0.8395 t_0.6-0.8=0.7262 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 163/299 (54% done) --- + [Epoch 163][10/66] loss=0.858965 avg=0.874521 VRAM=39.0GiB | 54.4% done | ETA(epoch): 1081s + [Epoch 163][20/66] loss=1.013647 avg=0.861871 VRAM=38.9GiB | 54.4% done | ETA(epoch): 889s + [Epoch 163][30/66] loss=0.896671 avg=0.869751 VRAM=39.0GiB | 54.5% done | ETA(epoch): 695s + [Epoch 163][40/66] loss=0.964204 avg=0.874810 VRAM=38.9GiB | 54.5% done | ETA(epoch): 502s + [Epoch 163][50/66] loss=0.784609 avg=0.864798 VRAM=39.0GiB | 54.6% done | ETA(epoch): 309s + [Epoch 163][60/66] loss=0.976221 avg=0.864000 VRAM=38.9GiB | 54.6% done | ETA(epoch): 116s + Train loss: 0.869469 (1274.9s) ETA: 3203min + Val loss: 0.836205 [t_0.0-0.2=1.0766 t_0.2-0.4=0.9648 t_0.4-0.6=0.8439 t_0.6-0.8=0.6988 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 164/299 (55% done) --- + [Epoch 164][10/66] loss=0.830222 avg=0.823156 VRAM=39.0GiB | 54.7% done | ETA(epoch): 1092s + [Epoch 164][20/66] loss=1.053601 avg=0.836610 VRAM=38.9GiB | 54.8% done | ETA(epoch): 893s + [Epoch 164][30/66] loss=0.886125 avg=0.860217 VRAM=39.0GiB | 54.8% done | ETA(epoch): 698s + [Epoch 164][40/66] loss=0.774833 avg=0.852711 VRAM=38.9GiB | 54.9% done | ETA(epoch): 504s + [Epoch 164][50/66] loss=0.737664 avg=0.848570 VRAM=39.0GiB | 54.9% done | ETA(epoch): 310s + [Epoch 164][60/66] loss=0.903797 avg=0.844459 VRAM=38.9GiB | 55.0% done | ETA(epoch): 116s + Train loss: 0.848816 (1277.2s) ETA: 3179min + Val loss: 0.844383 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9944 t_0.4-0.6=0.8099 t_0.6-0.8=0.7214 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 165/299 (55% done) --- + [Epoch 165][10/66] loss=0.930120 avg=0.863288 VRAM=39.0GiB | 55.1% done | ETA(epoch): 1082s + [Epoch 165][20/66] loss=0.935087 avg=0.860683 VRAM=38.9GiB | 55.1% done | ETA(epoch): 888s + [Epoch 165][30/66] loss=0.768294 avg=0.864092 VRAM=39.0GiB | 55.2% done | ETA(epoch): 695s + [Epoch 165][40/66] loss=0.917925 avg=0.867028 VRAM=38.9GiB | 55.2% done | ETA(epoch): 502s + [Epoch 165][50/66] loss=0.911939 avg=0.866552 VRAM=39.0GiB | 55.3% done | ETA(epoch): 309s + [Epoch 165][60/66] loss=0.881560 avg=0.872854 VRAM=38.9GiB | 55.3% done | ETA(epoch): 116s + Train loss: 0.872667 (1276.2s) ETA: 3155min + Val loss: 0.834524 [t_0.0-0.2=1.0847 t_0.2-0.4=0.9959 t_0.4-0.6=0.8434 t_0.6-0.8=0.7088 t_0.8-1.0=0.6334] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 166/299 (55% done) --- + [Epoch 166][10/66] loss=0.851224 avg=0.865900 VRAM=39.0GiB | 55.4% done | ETA(epoch): 1079s + [Epoch 166][20/66] loss=0.857175 avg=0.841330 VRAM=38.9GiB | 55.4% done | ETA(epoch): 887s + [Epoch 166][30/66] loss=0.843306 avg=0.836828 VRAM=39.0GiB | 55.5% done | ETA(epoch): 694s + [Epoch 166][40/66] loss=1.011784 avg=0.845079 VRAM=38.9GiB | 55.5% done | ETA(epoch): 502s + [Epoch 166][50/66] loss=0.889456 avg=0.845799 VRAM=39.0GiB | 55.6% done | ETA(epoch): 310s + [Epoch 166][60/66] loss=0.762287 avg=0.845526 VRAM=38.9GiB | 55.6% done | ETA(epoch): 116s + Train loss: 0.846893 (1277.5s) ETA: 3131min + Val loss: 0.864978 [t_0.0-0.2=1.0674 t_0.2-0.4=0.9688 t_0.4-0.6=0.8525 t_0.6-0.8=0.7007 t_0.8-1.0=0.6399] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 167/299 (56% done) --- + [Epoch 167][10/66] loss=0.834400 avg=0.819460 VRAM=39.0GiB | 55.7% done | ETA(epoch): 1080s + [Epoch 167][20/66] loss=0.895975 avg=0.819782 VRAM=38.9GiB | 55.8% done | ETA(epoch): 889s + [Epoch 167][30/66] loss=0.946078 avg=0.822445 VRAM=39.0GiB | 55.8% done | ETA(epoch): 696s + [Epoch 167][40/66] loss=0.790327 avg=0.817508 VRAM=38.9GiB | 55.9% done | ETA(epoch): 505s + [Epoch 167][50/66] loss=0.906226 avg=0.828887 VRAM=39.0GiB | 55.9% done | ETA(epoch): 311s + [Epoch 167][60/66] loss=0.684348 avg=0.830720 VRAM=38.9GiB | 56.0% done | ETA(epoch): 116s + Train loss: 0.834550 (1279.3s) ETA: 3107min + Val loss: 0.848796 [t_0.0-0.2=1.0668 t_0.2-0.4=1.0029 t_0.4-0.6=0.8454 t_0.6-0.8=0.7203 t_0.8-1.0=0.6355] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 168/299 (56% done) --- + [Epoch 168][10/66] loss=0.983216 avg=0.899046 VRAM=39.0GiB | 56.1% done | ETA(epoch): 1081s + [Epoch 168][20/66] loss=0.865449 avg=0.889658 VRAM=38.9GiB | 56.1% done | ETA(epoch): 888s + [Epoch 168][30/66] loss=0.868616 avg=0.861648 VRAM=39.0GiB | 56.2% done | ETA(epoch): 695s + [Epoch 168][40/66] loss=0.863018 avg=0.855459 VRAM=38.9GiB | 56.2% done | ETA(epoch): 504s + [Epoch 168][50/66] loss=0.874434 avg=0.851024 VRAM=39.0GiB | 56.3% done | ETA(epoch): 310s + [Epoch 168][60/66] loss=0.678020 avg=0.854729 VRAM=38.9GiB | 56.3% done | ETA(epoch): 116s + Train loss: 0.857587 (1277.6s) ETA: 3083min + Val loss: 0.838219 [t_0.0-0.2=1.0634 t_0.2-0.4=0.9908 t_0.4-0.6=0.8345 t_0.6-0.8=0.7149 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 19.3/188.5 GiB (10.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 169/299 (56% done) --- + [Epoch 169][10/66] loss=0.882875 avg=0.886498 VRAM=39.0GiB | 56.4% done | ETA(epoch): 1082s + [Epoch 169][20/66] loss=0.862229 avg=0.882900 VRAM=38.9GiB | 56.4% done | ETA(epoch): 889s + [Epoch 169][30/66] loss=0.916128 avg=0.875100 VRAM=39.0GiB | 56.5% done | ETA(epoch): 698s + [Epoch 169][40/66] loss=0.845212 avg=0.876271 VRAM=38.9GiB | 56.5% done | ETA(epoch): 504s + [Epoch 169][50/66] loss=0.781174 avg=0.870084 VRAM=39.0GiB | 56.6% done | ETA(epoch): 310s + [Epoch 169][60/66] loss=0.862417 avg=0.869828 VRAM=38.9GiB | 56.6% done | ETA(epoch): 116s + Train loss: 0.868858 (1278.2s) ETA: 3059min + Val loss: 0.858429 [t_0.0-0.2=1.0881 t_0.2-0.4=1.0040 t_0.4-0.6=0.8228 t_0.6-0.8=0.6906 t_0.8-1.0=0.6280] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 170/299 (57% done) --- + [Epoch 170][10/66] loss=0.813815 avg=0.855863 VRAM=39.0GiB | 56.7% done | ETA(epoch): 1079s + [Epoch 170][20/66] loss=0.922336 avg=0.862346 VRAM=38.9GiB | 56.8% done | ETA(epoch): 887s + [Epoch 170][30/66] loss=0.806417 avg=0.849920 VRAM=39.0GiB | 56.8% done | ETA(epoch): 695s + [Epoch 170][40/66] loss=0.953622 avg=0.849608 VRAM=38.9GiB | 56.9% done | ETA(epoch): 502s + [Epoch 170][50/66] loss=0.933001 avg=0.855353 VRAM=39.0GiB | 56.9% done | ETA(epoch): 309s + [Epoch 170][60/66] loss=0.809318 avg=0.851742 VRAM=38.9GiB | 57.0% done | ETA(epoch): 116s + Train loss: 0.850932 (1276.4s) ETA: 3035min + Val loss: 0.857869 [t_0.0-0.2=1.0548 t_0.2-0.4=1.0134 t_0.4-0.6=0.8496 t_0.6-0.8=0.7087 t_0.8-1.0=0.6283] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 171/299 (57% done) --- + [Epoch 171][10/66] loss=0.824045 avg=0.875599 VRAM=39.0GiB | 57.1% done | ETA(epoch): 1093s + [Epoch 171][20/66] loss=0.907838 avg=0.867631 VRAM=38.9GiB | 57.1% done | ETA(epoch): 893s + [Epoch 171][30/66] loss=0.969313 avg=0.864699 VRAM=39.0GiB | 57.2% done | ETA(epoch): 698s + [Epoch 171][40/66] loss=0.892482 avg=0.862621 VRAM=38.9GiB | 57.2% done | ETA(epoch): 504s + [Epoch 171][50/66] loss=0.843240 avg=0.861137 VRAM=39.0GiB | 57.3% done | ETA(epoch): 310s + [Epoch 171][60/66] loss=0.829461 avg=0.856880 VRAM=38.9GiB | 57.3% done | ETA(epoch): 116s + Train loss: 0.854978 (1279.5s) ETA: 3011min + Val loss: 0.864345 [t_0.0-0.2=1.0732 t_0.2-0.4=1.0092 t_0.4-0.6=0.8257 t_0.6-0.8=0.7247 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 172/299 (57% done) --- + [Epoch 172][10/66] loss=0.712111 avg=0.827898 VRAM=39.0GiB | 57.4% done | ETA(epoch): 1083s + [Epoch 172][20/66] loss=0.753432 avg=0.847371 VRAM=38.9GiB | 57.4% done | ETA(epoch): 890s + [Epoch 172][30/66] loss=0.829007 avg=0.841898 VRAM=39.0GiB | 57.5% done | ETA(epoch): 696s + [Epoch 172][40/66] loss=0.810562 avg=0.851344 VRAM=38.9GiB | 57.5% done | ETA(epoch): 503s + [Epoch 172][50/66] loss=0.785652 avg=0.854609 VRAM=39.0GiB | 57.6% done | ETA(epoch): 309s + [Epoch 172][60/66] loss=0.924375 avg=0.861852 VRAM=38.9GiB | 57.6% done | ETA(epoch): 116s + Train loss: 0.857100 (1278.4s) ETA: 2987min + Val loss: 0.844622 [t_0.0-0.2=1.0685 t_0.2-0.4=0.9964 t_0.4-0.6=0.8534 t_0.6-0.8=0.6822 t_0.8-1.0=0.6394] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 173/299 (58% done) --- + [Epoch 173][10/66] loss=0.770235 avg=0.883565 VRAM=39.0GiB | 57.7% done | ETA(epoch): 1080s + [Epoch 173][20/66] loss=0.800471 avg=0.838252 VRAM=38.9GiB | 57.8% done | ETA(epoch): 887s + [Epoch 173][30/66] loss=0.793432 avg=0.846192 VRAM=39.0GiB | 57.8% done | ETA(epoch): 695s + [Epoch 173][40/66] loss=0.882789 avg=0.835862 VRAM=38.9GiB | 57.9% done | ETA(epoch): 502s + [Epoch 173][50/66] loss=0.897870 avg=0.840352 VRAM=39.0GiB | 57.9% done | ETA(epoch): 311s + [Epoch 173][60/66] loss=0.756966 avg=0.835754 VRAM=38.9GiB | 58.0% done | ETA(epoch): 116s + Train loss: 0.840330 (1280.8s) ETA: 2963min + Val loss: 0.831774 [t_0.0-0.2=1.0921 t_0.2-0.4=0.9884 t_0.4-0.6=0.8404 t_0.6-0.8=0.6820 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 174/299 (58% done) --- + [Epoch 174][10/66] loss=0.897550 avg=0.880179 VRAM=39.0GiB | 58.1% done | ETA(epoch): 1080s + [Epoch 174][20/66] loss=0.950430 avg=0.882178 VRAM=38.9GiB | 58.1% done | ETA(epoch): 888s + [Epoch 174][30/66] loss=0.951184 avg=0.875280 VRAM=39.0GiB | 58.2% done | ETA(epoch): 695s + [Epoch 174][40/66] loss=0.858919 avg=0.874588 VRAM=38.9GiB | 58.2% done | ETA(epoch): 502s + [Epoch 174][50/66] loss=0.804110 avg=0.866124 VRAM=39.0GiB | 58.3% done | ETA(epoch): 309s + [Epoch 174][60/66] loss=0.881328 avg=0.861619 VRAM=38.9GiB | 58.3% done | ETA(epoch): 116s + Train loss: 0.859997 (1276.2s) ETA: 2939min + Val loss: 0.836600 [t_0.0-0.2=1.0639 t_0.2-0.4=0.9951 t_0.4-0.6=0.8755 t_0.6-0.8=0.7067 t_0.8-1.0=0.6336] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 175/299 (58% done) --- + [Epoch 175][10/66] loss=0.874702 avg=0.873494 VRAM=39.0GiB | 58.4% done | ETA(epoch): 1089s + [Epoch 175][20/66] loss=0.711593 avg=0.856093 VRAM=38.9GiB | 58.4% done | ETA(epoch): 894s + [Epoch 175][30/66] loss=0.794031 avg=0.855423 VRAM=39.0GiB | 58.5% done | ETA(epoch): 698s + [Epoch 175][40/66] loss=0.803300 avg=0.863127 VRAM=38.9GiB | 58.5% done | ETA(epoch): 504s + [Epoch 175][50/66] loss=0.813600 avg=0.858067 VRAM=39.0GiB | 58.6% done | ETA(epoch): 310s + [Epoch 175][60/66] loss=0.796194 avg=0.853725 VRAM=38.9GiB | 58.6% done | ETA(epoch): 116s + Train loss: 0.854556 (1278.4s) ETA: 2915min + Val loss: 0.853057 [t_0.0-0.2=1.0728 t_0.2-0.4=0.9864 t_0.4-0.6=0.8354 t_0.6-0.8=0.7095 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 176/299 (59% done) --- + [Epoch 176][10/66] loss=0.765460 avg=0.865388 VRAM=39.0GiB | 58.7% done | ETA(epoch): 1083s + [Epoch 176][20/66] loss=0.634453 avg=0.841974 VRAM=38.9GiB | 58.8% done | ETA(epoch): 889s + [Epoch 176][30/66] loss=0.816138 avg=0.846753 VRAM=39.0GiB | 58.8% done | ETA(epoch): 695s + [Epoch 176][40/66] loss=0.866259 avg=0.840352 VRAM=38.9GiB | 58.9% done | ETA(epoch): 502s + [Epoch 176][50/66] loss=0.822320 avg=0.841123 VRAM=39.0GiB | 58.9% done | ETA(epoch): 309s + [Epoch 176][60/66] loss=0.807308 avg=0.843577 VRAM=38.9GiB | 59.0% done | ETA(epoch): 116s + Train loss: 0.843958 (1275.6s) ETA: 2891min + Val loss: 0.862048 [t_0.0-0.2=1.0868 t_0.2-0.4=0.9759 t_0.4-0.6=0.8774 t_0.6-0.8=0.6893 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 177/299 (59% done) --- + [Epoch 177][10/66] loss=0.967264 avg=0.890443 VRAM=39.0GiB | 59.1% done | ETA(epoch): 1086s + [Epoch 177][20/66] loss=0.905585 avg=0.866114 VRAM=38.9GiB | 59.1% done | ETA(epoch): 896s + [Epoch 177][30/66] loss=0.826730 avg=0.859463 VRAM=39.0GiB | 59.2% done | ETA(epoch): 700s + [Epoch 177][40/66] loss=1.004294 avg=0.862138 VRAM=38.9GiB | 59.2% done | ETA(epoch): 505s + [Epoch 177][50/66] loss=0.855481 avg=0.859971 VRAM=39.0GiB | 59.3% done | ETA(epoch): 311s + [Epoch 177][60/66] loss=0.799947 avg=0.860003 VRAM=38.9GiB | 59.3% done | ETA(epoch): 116s + Train loss: 0.857742 (1280.4s) ETA: 2867min + Val loss: 0.841998 [t_0.0-0.2=1.0776 t_0.2-0.4=1.0010 t_0.4-0.6=0.8598 t_0.6-0.8=0.7009 t_0.8-1.0=0.6244] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 178/299 (59% done) --- + [Epoch 178][10/66] loss=0.880921 avg=0.864744 VRAM=39.0GiB | 59.4% done | ETA(epoch): 1084s + [Epoch 178][20/66] loss=0.977554 avg=0.856534 VRAM=38.9GiB | 59.4% done | ETA(epoch): 889s + [Epoch 178][30/66] loss=0.764527 avg=0.852391 VRAM=39.0GiB | 59.5% done | ETA(epoch): 697s + [Epoch 178][40/66] loss=0.889007 avg=0.849597 VRAM=38.9GiB | 59.5% done | ETA(epoch): 503s + [Epoch 178][50/66] loss=0.784663 avg=0.848752 VRAM=39.0GiB | 59.6% done | ETA(epoch): 310s + [Epoch 178][60/66] loss=0.841859 avg=0.853816 VRAM=38.9GiB | 59.6% done | ETA(epoch): 116s + Train loss: 0.855391 (1276.7s) ETA: 2843min + Val loss: 0.880403 [t_0.0-0.2=1.0794 t_0.2-0.4=1.0129 t_0.4-0.6=0.8492 t_0.6-0.8=0.7234 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 179/299 (60% done) --- + [Epoch 179][10/66] loss=0.842749 avg=0.866456 VRAM=39.0GiB | 59.7% done | ETA(epoch): 1084s + [Epoch 179][20/66] loss=0.867954 avg=0.862111 VRAM=38.9GiB | 59.8% done | ETA(epoch): 891s + [Epoch 179][30/66] loss=0.761764 avg=0.856427 VRAM=39.0GiB | 59.8% done | ETA(epoch): 696s + [Epoch 179][40/66] loss=0.855819 avg=0.855368 VRAM=38.9GiB | 59.9% done | ETA(epoch): 503s + [Epoch 179][50/66] loss=0.848803 avg=0.849154 VRAM=39.0GiB | 59.9% done | ETA(epoch): 309s + [Epoch 179][60/66] loss=0.784478 avg=0.851813 VRAM=38.9GiB | 60.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 11879 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 11879 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 11879 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 11879 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 11879 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 11879 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 11879 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 11879 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 11879 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 11879 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.854889 (2316.3s) ETA: 2831min + Val loss: 0.828922 [t_0.0-0.2=1.0825 t_0.2-0.4=0.9763 t_0.4-0.6=0.8611 t_0.6-0.8=0.6997 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 180/299 (60% done) --- + [Epoch 180][10/66] loss=0.955526 avg=0.831261 VRAM=39.0GiB | 60.1% done | ETA(epoch): 1081s + [Epoch 180][20/66] loss=0.702752 avg=0.844573 VRAM=38.9GiB | 60.1% done | ETA(epoch): 888s + [Epoch 180][30/66] loss=0.915869 avg=0.850785 VRAM=39.0GiB | 60.2% done | ETA(epoch): 695s + [Epoch 180][40/66] loss=0.766411 avg=0.844721 VRAM=38.9GiB | 60.2% done | ETA(epoch): 503s + [Epoch 180][50/66] loss=0.774481 avg=0.846539 VRAM=39.0GiB | 60.3% done | ETA(epoch): 309s + [Epoch 180][60/66] loss=0.857108 avg=0.848115 VRAM=38.9GiB | 60.3% done | ETA(epoch): 116s + Train loss: 0.845785 (1278.1s) ETA: 2807min + Val loss: 0.883896 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9888 t_0.4-0.6=0.8341 t_0.6-0.8=0.6988 t_0.8-1.0=0.6365] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 181/299 (60% done) --- + [Epoch 181][10/66] loss=0.905750 avg=0.860304 VRAM=39.0GiB | 60.4% done | ETA(epoch): 1080s + [Epoch 181][20/66] loss=0.826273 avg=0.849185 VRAM=38.9GiB | 60.4% done | ETA(epoch): 888s + [Epoch 181][30/66] loss=0.767820 avg=0.844361 VRAM=39.0GiB | 60.5% done | ETA(epoch): 695s + [Epoch 181][40/66] loss=0.899826 avg=0.849552 VRAM=38.9GiB | 60.5% done | ETA(epoch): 502s + [Epoch 181][50/66] loss=0.945853 avg=0.858445 VRAM=39.0GiB | 60.6% done | ETA(epoch): 309s + [Epoch 181][60/66] loss=1.065337 avg=0.858311 VRAM=38.9GiB | 60.6% done | ETA(epoch): 116s + Train loss: 0.854761 (1276.5s) ETA: 2783min + Val loss: 0.841813 [t_0.0-0.2=1.0837 t_0.2-0.4=0.9499 t_0.4-0.6=0.8326 t_0.6-0.8=0.7309 t_0.8-1.0=0.6235] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 182/299 (61% done) --- + [Epoch 182][10/66] loss=0.801813 avg=0.766331 VRAM=39.0GiB | 60.7% done | ETA(epoch): 1082s + [Epoch 182][20/66] loss=0.962912 avg=0.812498 VRAM=38.9GiB | 60.8% done | ETA(epoch): 888s + [Epoch 182][30/66] loss=0.811434 avg=0.815944 VRAM=39.0GiB | 60.8% done | ETA(epoch): 695s + [Epoch 182][40/66] loss=0.886885 avg=0.823651 VRAM=38.9GiB | 60.9% done | ETA(epoch): 502s + [Epoch 182][50/66] loss=0.845253 avg=0.822879 VRAM=39.0GiB | 60.9% done | ETA(epoch): 309s + [Epoch 182][60/66] loss=0.847214 avg=0.829820 VRAM=38.9GiB | 61.0% done | ETA(epoch): 116s + Train loss: 0.835022 (1276.7s) ETA: 2759min + Val loss: 0.839458 [t_0.0-0.2=1.0649 t_0.2-0.4=1.0059 t_0.4-0.6=0.8371 t_0.6-0.8=0.7052 t_0.8-1.0=0.6255] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 183/299 (61% done) --- + [Epoch 183][10/66] loss=0.928526 avg=0.828677 VRAM=39.0GiB | 61.1% done | ETA(epoch): 1080s + [Epoch 183][20/66] loss=0.811425 avg=0.821645 VRAM=38.9GiB | 61.1% done | ETA(epoch): 889s + [Epoch 183][30/66] loss=0.921650 avg=0.831460 VRAM=39.0GiB | 61.2% done | ETA(epoch): 696s + [Epoch 183][40/66] loss=0.749570 avg=0.829853 VRAM=38.9GiB | 61.2% done | ETA(epoch): 503s + [Epoch 183][50/66] loss=0.876007 avg=0.842617 VRAM=39.0GiB | 61.3% done | ETA(epoch): 310s + [Epoch 183][60/66] loss=0.725831 avg=0.840332 VRAM=38.9GiB | 61.3% done | ETA(epoch): 116s + Train loss: 0.843082 (1277.3s) ETA: 2735min + Val loss: 0.817016 [t_0.0-0.2=1.0770 t_0.2-0.4=0.9913 t_0.4-0.6=0.8560 t_0.6-0.8=0.7059 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 184/299 (61% done) --- + [Epoch 184][10/66] loss=0.954278 avg=0.859244 VRAM=39.0GiB | 61.4% done | ETA(epoch): 1082s + [Epoch 184][20/66] loss=0.815655 avg=0.845368 VRAM=38.9GiB | 61.4% done | ETA(epoch): 889s + [Epoch 184][30/66] loss=0.826385 avg=0.851210 VRAM=39.0GiB | 61.5% done | ETA(epoch): 696s + [Epoch 184][40/66] loss=0.740888 avg=0.853907 VRAM=38.9GiB | 61.5% done | ETA(epoch): 502s + [Epoch 184][50/66] loss=0.831948 avg=0.850127 VRAM=39.0GiB | 61.6% done | ETA(epoch): 309s + [Epoch 184][60/66] loss=0.912180 avg=0.856127 VRAM=38.9GiB | 61.6% done | ETA(epoch): 116s + Train loss: 0.858167 (1275.3s) ETA: 2711min + Val loss: 0.830857 [t_0.0-0.2=1.0599 t_0.2-0.4=0.9777 t_0.4-0.6=0.8450 t_0.6-0.8=0.7205 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 185/299 (62% done) --- + [Epoch 185][10/66] loss=0.787074 avg=0.795025 VRAM=39.0GiB | 61.7% done | ETA(epoch): 1081s + [Epoch 185][20/66] loss=0.770874 avg=0.812289 VRAM=38.9GiB | 61.8% done | ETA(epoch): 888s + [Epoch 185][30/66] loss=0.815088 avg=0.826218 VRAM=39.0GiB | 61.8% done | ETA(epoch): 695s + [Epoch 185][40/66] loss=0.975460 avg=0.838121 VRAM=38.9GiB | 61.9% done | ETA(epoch): 503s + [Epoch 185][50/66] loss=0.978223 avg=0.838267 VRAM=39.0GiB | 61.9% done | ETA(epoch): 309s + [Epoch 185][60/66] loss=0.806644 avg=0.836377 VRAM=38.9GiB | 62.0% done | ETA(epoch): 116s + Train loss: 0.836677 (1275.8s) ETA: 2687min + Val loss: 0.868443 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9778 t_0.4-0.6=0.8622 t_0.6-0.8=0.7498 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 186/299 (62% done) --- + [Epoch 186][10/66] loss=0.838669 avg=0.863289 VRAM=39.0GiB | 62.1% done | ETA(epoch): 1104s + [Epoch 186][20/66] loss=0.785010 avg=0.842901 VRAM=38.9GiB | 62.1% done | ETA(epoch): 898s + [Epoch 186][30/66] loss=0.966401 avg=0.848004 VRAM=39.0GiB | 62.2% done | ETA(epoch): 701s + [Epoch 186][40/66] loss=0.819613 avg=0.849025 VRAM=38.9GiB | 62.2% done | ETA(epoch): 505s + [Epoch 186][50/66] loss=0.844975 avg=0.844930 VRAM=39.0GiB | 62.3% done | ETA(epoch): 311s + [Epoch 186][60/66] loss=0.851712 avg=0.848399 VRAM=38.9GiB | 62.3% done | ETA(epoch): 117s + Train loss: 0.847808 (1281.2s) ETA: 2663min + Val loss: 0.845171 [t_0.0-0.2=1.0674 t_0.2-0.4=0.9968 t_0.4-0.6=0.8464 t_0.6-0.8=0.7180 t_0.8-1.0=0.6367] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 187/299 (62% done) --- + [Epoch 187][10/66] loss=0.994742 avg=0.888487 VRAM=39.0GiB | 62.4% done | ETA(epoch): 1082s + [Epoch 187][20/66] loss=0.864994 avg=0.866576 VRAM=38.9GiB | 62.4% done | ETA(epoch): 889s + [Epoch 187][30/66] loss=0.877323 avg=0.865847 VRAM=39.0GiB | 62.5% done | ETA(epoch): 697s + [Epoch 187][40/66] loss=0.736655 avg=0.849852 VRAM=38.9GiB | 62.5% done | ETA(epoch): 503s + [Epoch 187][50/66] loss=0.701235 avg=0.848689 VRAM=39.0GiB | 62.6% done | ETA(epoch): 310s + [Epoch 187][60/66] loss=0.908854 avg=0.847946 VRAM=38.9GiB | 62.6% done | ETA(epoch): 116s + Train loss: 0.852658 (1277.2s) ETA: 2639min + Val loss: 0.846112 [t_0.0-0.2=1.0805 t_0.2-0.4=0.9882 t_0.4-0.6=0.8061 t_0.6-0.8=0.7156 t_0.8-1.0=0.6170] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 188/299 (63% done) --- + [Epoch 188][10/66] loss=0.851497 avg=0.883747 VRAM=39.0GiB | 62.7% done | ETA(epoch): 1083s + [Epoch 188][20/66] loss=0.757507 avg=0.876495 VRAM=38.9GiB | 62.8% done | ETA(epoch): 890s + [Epoch 188][30/66] loss=0.743337 avg=0.874577 VRAM=39.0GiB | 62.8% done | ETA(epoch): 696s + [Epoch 188][40/66] loss=0.791038 avg=0.863223 VRAM=38.9GiB | 62.9% done | ETA(epoch): 503s + [Epoch 188][50/66] loss=0.951596 avg=0.855261 VRAM=39.0GiB | 62.9% done | ETA(epoch): 309s + [Epoch 188][60/66] loss=0.977455 avg=0.862945 VRAM=38.9GiB | 63.0% done | ETA(epoch): 116s + Train loss: 0.858187 (1278.6s) ETA: 2615min + Val loss: 0.846513 [t_0.0-0.2=1.0618 t_0.2-0.4=0.9980 t_0.4-0.6=0.8656 t_0.6-0.8=0.7238 t_0.8-1.0=0.6436] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 189/299 (63% done) --- + [Epoch 189][10/66] loss=0.879649 avg=0.866260 VRAM=39.0GiB | 63.1% done | ETA(epoch): 1081s + [Epoch 189][20/66] loss=0.964780 avg=0.880830 VRAM=38.9GiB | 63.1% done | ETA(epoch): 889s + [Epoch 189][30/66] loss=0.893995 avg=0.867222 VRAM=39.0GiB | 63.2% done | ETA(epoch): 696s + [Epoch 189][40/66] loss=0.735289 avg=0.852072 VRAM=38.9GiB | 63.2% done | ETA(epoch): 503s + [Epoch 189][50/66] loss=0.855206 avg=0.851309 VRAM=39.0GiB | 63.3% done | ETA(epoch): 311s + [Epoch 189][60/66] loss=0.847039 avg=0.857090 VRAM=38.9GiB | 63.3% done | ETA(epoch): 116s + Train loss: 0.855083 (1280.0s) ETA: 2591min + Val loss: 0.848205 [t_0.0-0.2=1.0658 t_0.2-0.4=0.9903 t_0.4-0.6=0.8744 t_0.6-0.8=0.6919 t_0.8-1.0=0.6314] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 190/299 (63% done) --- + [Epoch 190][10/66] loss=0.862242 avg=0.867424 VRAM=39.0GiB | 63.4% done | ETA(epoch): 1082s + [Epoch 190][20/66] loss=0.984699 avg=0.885725 VRAM=38.9GiB | 63.4% done | ETA(epoch): 889s + [Epoch 190][30/66] loss=0.811139 avg=0.871465 VRAM=39.0GiB | 63.5% done | ETA(epoch): 695s + [Epoch 190][40/66] loss=0.936648 avg=0.863492 VRAM=38.9GiB | 63.5% done | ETA(epoch): 504s + [Epoch 190][50/66] loss=0.769367 avg=0.851429 VRAM=39.0GiB | 63.6% done | ETA(epoch): 310s + [Epoch 190][60/66] loss=0.936686 avg=0.854845 VRAM=38.9GiB | 63.6% done | ETA(epoch): 116s + Train loss: 0.853740 (1278.2s) ETA: 2567min + Val loss: 0.858427 [t_0.0-0.2=1.0624 t_0.2-0.4=0.9999 t_0.4-0.6=0.8147 t_0.6-0.8=0.7148 t_0.8-1.0=0.6193] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 191/299 (64% done) --- + [Epoch 191][10/66] loss=0.661753 avg=0.813730 VRAM=39.0GiB | 63.7% done | ETA(epoch): 1079s + [Epoch 191][20/66] loss=0.856452 avg=0.836753 VRAM=38.9GiB | 63.8% done | ETA(epoch): 888s + [Epoch 191][30/66] loss=0.901083 avg=0.832861 VRAM=39.0GiB | 63.8% done | ETA(epoch): 697s + [Epoch 191][40/66] loss=0.819431 avg=0.837441 VRAM=38.9GiB | 63.9% done | ETA(epoch): 503s + [Epoch 191][50/66] loss=0.898289 avg=0.842361 VRAM=39.0GiB | 63.9% done | ETA(epoch): 309s + [Epoch 191][60/66] loss=0.823730 avg=0.843735 VRAM=38.9GiB | 64.0% done | ETA(epoch): 116s + Train loss: 0.845628 (1277.3s) ETA: 2543min + Val loss: 0.879769 [t_0.0-0.2=1.0759 t_0.2-0.4=0.9990 t_0.4-0.6=0.8569 t_0.6-0.8=0.6970 t_0.8-1.0=0.6049] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0191 + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 192/299 (64% done) --- + [Epoch 192][10/66] loss=0.952834 avg=0.895641 VRAM=39.0GiB | 64.1% done | ETA(epoch): 1082s + [Epoch 192][20/66] loss=0.793768 avg=0.868429 VRAM=38.9GiB | 64.1% done | ETA(epoch): 889s + [Epoch 192][30/66] loss=0.888270 avg=0.860603 VRAM=39.0GiB | 64.2% done | ETA(epoch): 695s + [Epoch 192][40/66] loss=0.924702 avg=0.865649 VRAM=38.9GiB | 64.2% done | ETA(epoch): 502s + [Epoch 192][50/66] loss=0.758061 avg=0.859826 VRAM=39.0GiB | 64.3% done | ETA(epoch): 309s + [Epoch 192][60/66] loss=0.928614 avg=0.857959 VRAM=38.9GiB | 64.3% done | ETA(epoch): 116s + Train loss: 0.854300 (1274.8s) ETA: 2519min + Val loss: 0.862768 [t_0.0-0.2=1.0719 t_0.2-0.4=0.9905 t_0.4-0.6=0.8426 t_0.6-0.8=0.7101 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 193/299 (64% done) --- + [Epoch 193][10/66] loss=0.783454 avg=0.799377 VRAM=39.0GiB | 64.4% done | ETA(epoch): 1079s + [Epoch 193][20/66] loss=0.963347 avg=0.845816 VRAM=38.9GiB | 64.4% done | ETA(epoch): 887s + [Epoch 193][30/66] loss=0.762156 avg=0.842637 VRAM=39.0GiB | 64.5% done | ETA(epoch): 695s + [Epoch 193][40/66] loss=0.901872 avg=0.851323 VRAM=38.9GiB | 64.5% done | ETA(epoch): 502s + [Epoch 193][50/66] loss=0.878993 avg=0.856734 VRAM=39.0GiB | 64.6% done | ETA(epoch): 309s + [Epoch 193][60/66] loss=0.807156 avg=0.863112 VRAM=38.9GiB | 64.6% done | ETA(epoch): 116s + Train loss: 0.864079 (1274.6s) ETA: 2495min + Val loss: 0.871383 [t_0.0-0.2=1.0761 t_0.2-0.4=1.0209 t_0.4-0.6=0.8448 t_0.6-0.8=0.7128 t_0.8-1.0=0.6352] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 194/299 (65% done) --- + [Epoch 194][10/66] loss=0.896008 avg=0.863966 VRAM=39.0GiB | 64.7% done | ETA(epoch): 1080s + [Epoch 194][20/66] loss=0.851513 avg=0.868040 VRAM=38.9GiB | 64.8% done | ETA(epoch): 889s + [Epoch 194][30/66] loss=0.972919 avg=0.856188 VRAM=39.0GiB | 64.8% done | ETA(epoch): 695s + [Epoch 194][40/66] loss=0.905823 avg=0.863763 VRAM=38.9GiB | 64.9% done | ETA(epoch): 502s + [Epoch 194][50/66] loss=0.888146 avg=0.864894 VRAM=39.0GiB | 64.9% done | ETA(epoch): 309s + [Epoch 194][60/66] loss=0.820534 avg=0.866387 VRAM=38.9GiB | 65.0% done | ETA(epoch): 116s + Train loss: 0.870530 (1275.7s) ETA: 2471min + Val loss: 0.873068 [t_0.0-0.2=1.0637 t_0.2-0.4=1.0131 t_0.4-0.6=0.8503 t_0.6-0.8=0.7263 t_0.8-1.0=0.6206] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 195/299 (65% done) --- + [Epoch 195][10/66] loss=0.907474 avg=0.854357 VRAM=39.0GiB | 65.1% done | ETA(epoch): 1079s + [Epoch 195][20/66] loss=0.775240 avg=0.860957 VRAM=38.9GiB | 65.1% done | ETA(epoch): 887s + [Epoch 195][30/66] loss=0.749579 avg=0.870805 VRAM=39.0GiB | 65.2% done | ETA(epoch): 695s + [Epoch 195][40/66] loss=0.849233 avg=0.879916 VRAM=38.9GiB | 65.2% done | ETA(epoch): 502s + [Epoch 195][50/66] loss=0.778502 avg=0.872784 VRAM=39.0GiB | 65.3% done | ETA(epoch): 309s + [Epoch 195][60/66] loss=0.807191 avg=0.867680 VRAM=38.9GiB | 65.3% done | ETA(epoch): 116s + Train loss: 0.864088 (1275.1s) ETA: 2447min + Val loss: 0.830335 [t_0.0-0.2=1.0787 t_0.2-0.4=0.9959 t_0.4-0.6=0.8444 t_0.6-0.8=0.7299 t_0.8-1.0=0.6148] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 196/299 (65% done) --- + [Epoch 196][10/66] loss=0.776203 avg=0.859628 VRAM=39.0GiB | 65.4% done | ETA(epoch): 1081s + [Epoch 196][20/66] loss=0.776832 avg=0.839480 VRAM=38.9GiB | 65.4% done | ETA(epoch): 888s + [Epoch 196][30/66] loss=0.815374 avg=0.845299 VRAM=39.0GiB | 65.5% done | ETA(epoch): 696s + [Epoch 196][40/66] loss=0.935524 avg=0.847831 VRAM=38.9GiB | 65.5% done | ETA(epoch): 503s + [Epoch 196][50/66] loss=0.973109 avg=0.859281 VRAM=39.0GiB | 65.6% done | ETA(epoch): 309s + [Epoch 196][60/66] loss=0.904651 avg=0.857267 VRAM=38.9GiB | 65.6% done | ETA(epoch): 116s + Train loss: 0.855664 (1276.1s) ETA: 2424min + Val loss: 0.840250 [t_0.0-0.2=1.0506 t_0.2-0.4=1.0045 t_0.4-0.6=0.8453 t_0.6-0.8=0.6967 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 197/299 (66% done) --- + [Epoch 197][10/66] loss=0.656708 avg=0.813522 VRAM=39.0GiB | 65.7% done | ETA(epoch): 1082s + [Epoch 197][20/66] loss=0.847798 avg=0.841406 VRAM=38.9GiB | 65.8% done | ETA(epoch): 889s + [Epoch 197][30/66] loss=0.988837 avg=0.840915 VRAM=39.0GiB | 65.8% done | ETA(epoch): 698s + [Epoch 197][40/66] loss=0.751776 avg=0.853121 VRAM=38.9GiB | 65.9% done | ETA(epoch): 504s + [Epoch 197][50/66] loss=0.885949 avg=0.859875 VRAM=39.0GiB | 65.9% done | ETA(epoch): 310s + [Epoch 197][60/66] loss=0.911105 avg=0.858602 VRAM=38.9GiB | 66.0% done | ETA(epoch): 116s + Train loss: 0.856545 (1278.4s) ETA: 2400min + Val loss: 0.867221 [t_0.0-0.2=1.0799 t_0.2-0.4=1.0116 t_0.4-0.6=0.8128 t_0.6-0.8=0.6961 t_0.8-1.0=0.6322] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 198/299 (66% done) --- + [Epoch 198][10/66] loss=0.797354 avg=0.842461 VRAM=39.0GiB | 66.1% done | ETA(epoch): 1084s + [Epoch 198][20/66] loss=0.993424 avg=0.858235 VRAM=38.9GiB | 66.1% done | ETA(epoch): 890s + [Epoch 198][30/66] loss=0.897602 avg=0.851223 VRAM=39.0GiB | 66.2% done | ETA(epoch): 696s + [Epoch 198][40/66] loss=0.984334 avg=0.859647 VRAM=38.9GiB | 66.2% done | ETA(epoch): 503s + [Epoch 198][50/66] loss=0.718930 avg=0.862990 VRAM=39.0GiB | 66.3% done | ETA(epoch): 309s + [Epoch 198][60/66] loss=0.845650 avg=0.853278 VRAM=38.9GiB | 66.3% done | ETA(epoch): 116s + Train loss: 0.856547 (1276.6s) ETA: 2376min + Val loss: 0.863740 [t_0.0-0.2=1.0540 t_0.2-0.4=1.0190 t_0.4-0.6=0.8243 t_0.6-0.8=0.7235 t_0.8-1.0=0.6195] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 199/299 (66% done) --- + [Epoch 199][10/66] loss=0.923377 avg=0.867613 VRAM=39.0GiB | 66.4% done | ETA(epoch): 1082s + [Epoch 199][20/66] loss=0.794157 avg=0.852217 VRAM=38.9GiB | 66.4% done | ETA(epoch): 890s + [Epoch 199][30/66] loss=0.828963 avg=0.865932 VRAM=39.0GiB | 66.5% done | ETA(epoch): 697s + [Epoch 199][40/66] loss=0.873338 avg=0.854746 VRAM=38.9GiB | 66.5% done | ETA(epoch): 503s + [Epoch 199][50/66] loss=0.923502 avg=0.852822 VRAM=39.0GiB | 66.6% done | ETA(epoch): 310s + [Epoch 199][60/66] loss=0.936611 avg=0.849942 VRAM=38.9GiB | 66.6% done | ETA(epoch): 116s + Train loss: 0.846586 (1276.7s) ETA: 2352min + Val loss: 0.839559 [t_0.0-0.2=1.0616 t_0.2-0.4=0.9901 t_0.4-0.6=0.8425 t_0.6-0.8=0.6936 t_0.8-1.0=0.6415] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 200/299 (67% done) --- + [Epoch 200][10/66] loss=0.736022 avg=0.843361 VRAM=39.0GiB | 66.7% done | ETA(epoch): 1079s + [Epoch 200][20/66] loss=0.839646 avg=0.851721 VRAM=38.9GiB | 66.8% done | ETA(epoch): 888s + [Epoch 200][30/66] loss=0.889810 avg=0.845446 VRAM=39.0GiB | 66.8% done | ETA(epoch): 696s + [Epoch 200][40/66] loss=0.876579 avg=0.842379 VRAM=38.9GiB | 66.9% done | ETA(epoch): 502s + [Epoch 200][50/66] loss=0.912024 avg=0.843563 VRAM=39.0GiB | 66.9% done | ETA(epoch): 309s + [Epoch 200][60/66] loss=0.841673 avg=0.848148 VRAM=38.9GiB | 67.0% done | ETA(epoch): 116s + Train loss: 0.846321 (1275.9s) ETA: 2328min + Val loss: 0.868993 [t_0.0-0.2=1.0812 t_0.2-0.4=1.0152 t_0.4-0.6=0.8458 t_0.6-0.8=0.7032 t_0.8-1.0=0.6297] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 201/299 (67% done) --- + [Epoch 201][10/66] loss=0.896975 avg=0.846270 VRAM=39.0GiB | 67.1% done | ETA(epoch): 1099s + [Epoch 201][20/66] loss=0.702408 avg=0.839685 VRAM=38.9GiB | 67.1% done | ETA(epoch): 896s + [Epoch 201][30/66] loss=0.868895 avg=0.853357 VRAM=39.0GiB | 67.2% done | ETA(epoch): 700s + [Epoch 201][40/66] loss=0.951307 avg=0.859061 VRAM=38.9GiB | 67.2% done | ETA(epoch): 505s + [Epoch 201][50/66] loss=0.854910 avg=0.855916 VRAM=39.0GiB | 67.3% done | ETA(epoch): 310s + [Epoch 201][60/66] loss=0.903897 avg=0.853148 VRAM=38.9GiB | 67.3% done | ETA(epoch): 117s + Train loss: 0.852701 (1281.4s) ETA: 2304min + Val loss: 0.850986 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0028 t_0.4-0.6=0.8309 t_0.6-0.8=0.7121 t_0.8-1.0=0.6261] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 202/299 (67% done) --- + [Epoch 202][10/66] loss=0.884982 avg=0.858119 VRAM=39.0GiB | 67.4% done | ETA(epoch): 1083s + [Epoch 202][20/66] loss=0.905605 avg=0.852859 VRAM=38.9GiB | 67.4% done | ETA(epoch): 889s + [Epoch 202][30/66] loss=0.832973 avg=0.852862 VRAM=39.0GiB | 67.5% done | ETA(epoch): 695s + [Epoch 202][40/66] loss=0.755365 avg=0.850523 VRAM=38.9GiB | 67.5% done | ETA(epoch): 502s + [Epoch 202][50/66] loss=0.935959 avg=0.848706 VRAM=39.0GiB | 67.6% done | ETA(epoch): 309s + [Epoch 202][60/66] loss=0.773954 avg=0.846744 VRAM=38.9GiB | 67.6% done | ETA(epoch): 116s + Train loss: 0.837717 (1277.7s) ETA: 2281min + Val loss: 0.826683 [t_0.0-0.2=1.0834 t_0.2-0.4=0.9670 t_0.4-0.6=0.8411 t_0.6-0.8=0.7261 t_0.8-1.0=0.6203] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 203/299 (68% done) --- + [Epoch 203][10/66] loss=0.963331 avg=0.851696 VRAM=39.0GiB | 67.7% done | ETA(epoch): 1082s + [Epoch 203][20/66] loss=0.753083 avg=0.833913 VRAM=38.9GiB | 67.8% done | ETA(epoch): 890s + [Epoch 203][30/66] loss=0.747661 avg=0.834022 VRAM=39.0GiB | 67.8% done | ETA(epoch): 696s + [Epoch 203][40/66] loss=0.744961 avg=0.838427 VRAM=38.9GiB | 67.9% done | ETA(epoch): 502s + [Epoch 203][50/66] loss=0.760276 avg=0.843191 VRAM=39.0GiB | 67.9% done | ETA(epoch): 310s + [Epoch 203][60/66] loss=0.944491 avg=0.848694 VRAM=38.9GiB | 68.0% done | ETA(epoch): 116s + Train loss: 0.847004 (1277.8s) ETA: 2257min + Val loss: 0.891947 [t_0.0-0.2=1.0606 t_0.2-0.4=0.9922 t_0.4-0.6=0.8504 t_0.6-0.8=0.7212 t_0.8-1.0=0.6321] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 204/299 (68% done) --- + [Epoch 204][10/66] loss=0.778110 avg=0.880190 VRAM=39.0GiB | 68.1% done | ETA(epoch): 1082s + [Epoch 204][20/66] loss=0.831184 avg=0.860399 VRAM=38.9GiB | 68.1% done | ETA(epoch): 889s + [Epoch 204][30/66] loss=0.981502 avg=0.865437 VRAM=39.0GiB | 68.2% done | ETA(epoch): 696s + [Epoch 204][40/66] loss=0.824911 avg=0.859004 VRAM=38.9GiB | 68.2% done | ETA(epoch): 502s + [Epoch 204][50/66] loss=0.929950 avg=0.862460 VRAM=39.0GiB | 68.3% done | ETA(epoch): 309s + [Epoch 204][60/66] loss=0.865313 avg=0.860148 VRAM=38.9GiB | 68.3% done | ETA(epoch): 116s + Train loss: 0.861096 (1276.2s) ETA: 2233min + Val loss: 0.863805 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0005 t_0.4-0.6=0.8569 t_0.6-0.8=0.6738 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 205/299 (68% done) --- + [Epoch 205][10/66] loss=0.827914 avg=0.853771 VRAM=39.0GiB | 68.4% done | ETA(epoch): 1080s + [Epoch 205][20/66] loss=0.880222 avg=0.840777 VRAM=38.9GiB | 68.4% done | ETA(epoch): 888s + [Epoch 205][30/66] loss=0.867302 avg=0.838538 VRAM=39.0GiB | 68.5% done | ETA(epoch): 700s + [Epoch 205][40/66] loss=0.890506 avg=0.840379 VRAM=38.9GiB | 68.5% done | ETA(epoch): 505s + [Epoch 205][50/66] loss=0.743661 avg=0.848468 VRAM=39.0GiB | 68.6% done | ETA(epoch): 311s + [Epoch 205][60/66] loss=0.993610 avg=0.848924 VRAM=38.9GiB | 68.6% done | ETA(epoch): 116s + Train loss: 0.846678 (1279.9s) ETA: 2209min + Val loss: 0.825014 [t_0.0-0.2=1.0818 t_0.2-0.4=0.9991 t_0.4-0.6=0.8265 t_0.6-0.8=0.7056 t_0.8-1.0=0.6210] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 206/299 (69% done) --- + [Epoch 206][10/66] loss=0.872641 avg=0.844022 VRAM=39.0GiB | 68.7% done | ETA(epoch): 1081s + [Epoch 206][20/66] loss=0.928892 avg=0.838924 VRAM=38.9GiB | 68.8% done | ETA(epoch): 896s + [Epoch 206][30/66] loss=0.913428 avg=0.844190 VRAM=39.0GiB | 68.8% done | ETA(epoch): 699s + [Epoch 206][40/66] loss=0.758776 avg=0.837969 VRAM=38.9GiB | 68.9% done | ETA(epoch): 504s + [Epoch 206][50/66] loss=0.776534 avg=0.841818 VRAM=39.0GiB | 68.9% done | ETA(epoch): 310s + [Epoch 206][60/66] loss=0.770347 avg=0.849563 VRAM=38.9GiB | 69.0% done | ETA(epoch): 116s + Train loss: 0.851875 (1278.4s) ETA: 2185min + Val loss: 0.863510 [t_0.0-0.2=1.0799 t_0.2-0.4=1.0025 t_0.4-0.6=0.8661 t_0.6-0.8=0.7261 t_0.8-1.0=0.6274] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 207/299 (69% done) --- + [Epoch 207][10/66] loss=0.736417 avg=0.826944 VRAM=39.0GiB | 69.1% done | ETA(epoch): 1083s + [Epoch 207][20/66] loss=0.828265 avg=0.835686 VRAM=38.9GiB | 69.1% done | ETA(epoch): 896s + [Epoch 207][30/66] loss=0.847810 avg=0.840233 VRAM=39.0GiB | 69.2% done | ETA(epoch): 699s + [Epoch 207][40/66] loss=0.829466 avg=0.842920 VRAM=38.9GiB | 69.2% done | ETA(epoch): 505s + [Epoch 207][50/66] loss=0.855904 avg=0.844929 VRAM=39.0GiB | 69.3% done | ETA(epoch): 310s + [Epoch 207][60/66] loss=0.919939 avg=0.848950 VRAM=38.9GiB | 69.3% done | ETA(epoch): 116s + Train loss: 0.845455 (1278.5s) ETA: 2162min + Val loss: 0.850309 [t_0.0-0.2=1.0540 t_0.2-0.4=1.0009 t_0.4-0.6=0.8459 t_0.6-0.8=0.7067 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 208/299 (69% done) --- + [Epoch 208][10/66] loss=0.862193 avg=0.889014 VRAM=39.0GiB | 69.4% done | ETA(epoch): 1085s + [Epoch 208][20/66] loss=0.820143 avg=0.869064 VRAM=38.9GiB | 69.4% done | ETA(epoch): 892s + [Epoch 208][30/66] loss=0.855166 avg=0.860708 VRAM=39.0GiB | 69.5% done | ETA(epoch): 698s + [Epoch 208][40/66] loss=0.873401 avg=0.843044 VRAM=38.9GiB | 69.5% done | ETA(epoch): 503s + [Epoch 208][50/66] loss=0.839277 avg=0.845299 VRAM=39.0GiB | 69.6% done | ETA(epoch): 310s + [Epoch 208][60/66] loss=0.821072 avg=0.845321 VRAM=38.9GiB | 69.6% done | ETA(epoch): 116s + Train loss: 0.848461 (1277.5s) ETA: 2138min + Val loss: 0.842706 [t_0.0-0.2=1.0551 t_0.2-0.4=1.0040 t_0.4-0.6=0.8575 t_0.6-0.8=0.7051 t_0.8-1.0=0.6303] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 209/299 (70% done) --- + [Epoch 209][10/66] loss=0.933747 avg=0.911236 VRAM=39.0GiB | 69.7% done | ETA(epoch): 1079s + [Epoch 209][20/66] loss=0.901037 avg=0.867295 VRAM=38.9GiB | 69.8% done | ETA(epoch): 888s + [Epoch 209][30/66] loss=0.717567 avg=0.847153 VRAM=39.0GiB | 69.8% done | ETA(epoch): 695s + [Epoch 209][40/66] loss=0.841785 avg=0.840135 VRAM=38.9GiB | 69.9% done | ETA(epoch): 502s + [Epoch 209][50/66] loss=0.692303 avg=0.839365 VRAM=39.0GiB | 69.9% done | ETA(epoch): 309s + [Epoch 209][60/66] loss=0.753920 avg=0.836326 VRAM=38.9GiB | 70.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 13859 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 13859 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 13859 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 13859 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 13859 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 13859 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 13859 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 13859 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 13859 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 13859 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.844830 (2318.7s) ETA: 2122min + Val loss: 0.845076 [t_0.0-0.2=1.0664 t_0.2-0.4=1.0216 t_0.4-0.6=0.8350 t_0.6-0.8=0.7024 t_0.8-1.0=0.6152] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 210/299 (70% done) --- + [Epoch 210][10/66] loss=0.930475 avg=0.883703 VRAM=39.0GiB | 70.1% done | ETA(epoch): 1081s + [Epoch 210][20/66] loss=0.749998 avg=0.853664 VRAM=38.9GiB | 70.1% done | ETA(epoch): 891s + [Epoch 210][30/66] loss=0.760376 avg=0.842110 VRAM=39.0GiB | 70.2% done | ETA(epoch): 697s + [Epoch 210][40/66] loss=0.778933 avg=0.850137 VRAM=38.9GiB | 70.2% done | ETA(epoch): 503s + [Epoch 210][50/66] loss=0.772221 avg=0.856154 VRAM=39.0GiB | 70.3% done | ETA(epoch): 310s + [Epoch 210][60/66] loss=0.888280 avg=0.852249 VRAM=38.9GiB | 70.3% done | ETA(epoch): 116s + Train loss: 0.849448 (1278.5s) ETA: 2098min + Val loss: 0.807952 [t_0.0-0.2=1.0634 t_0.2-0.4=1.0196 t_0.4-0.6=0.8252 t_0.6-0.8=0.7047 t_0.8-1.0=0.6354] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 211/299 (70% done) --- + [Epoch 211][10/66] loss=0.813425 avg=0.871564 VRAM=39.0GiB | 70.4% done | ETA(epoch): 1080s + [Epoch 211][20/66] loss=0.878340 avg=0.851420 VRAM=38.9GiB | 70.4% done | ETA(epoch): 891s + [Epoch 211][30/66] loss=0.896246 avg=0.859193 VRAM=39.0GiB | 70.5% done | ETA(epoch): 697s + [Epoch 211][40/66] loss=1.005399 avg=0.855394 VRAM=38.9GiB | 70.5% done | ETA(epoch): 503s + [Epoch 211][50/66] loss=0.895282 avg=0.858452 VRAM=39.0GiB | 70.6% done | ETA(epoch): 310s + [Epoch 211][60/66] loss=0.811803 avg=0.862947 VRAM=38.9GiB | 70.6% done | ETA(epoch): 116s + Train loss: 0.860779 (1277.6s) ETA: 2074min + Val loss: 0.844125 [t_0.0-0.2=1.0762 t_0.2-0.4=1.0242 t_0.4-0.6=0.8294 t_0.6-0.8=0.6864 t_0.8-1.0=0.6222] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 212/299 (71% done) --- + [Epoch 212][10/66] loss=0.928109 avg=0.846213 VRAM=39.0GiB | 70.7% done | ETA(epoch): 1081s + [Epoch 212][20/66] loss=0.871140 avg=0.844269 VRAM=38.9GiB | 70.8% done | ETA(epoch): 889s + [Epoch 212][30/66] loss=0.887548 avg=0.838426 VRAM=39.0GiB | 70.8% done | ETA(epoch): 696s + [Epoch 212][40/66] loss=0.996483 avg=0.847382 VRAM=38.9GiB | 70.9% done | ETA(epoch): 504s + [Epoch 212][50/66] loss=0.857023 avg=0.837277 VRAM=39.0GiB | 70.9% done | ETA(epoch): 310s + [Epoch 212][60/66] loss=0.803226 avg=0.828786 VRAM=38.9GiB | 71.0% done | ETA(epoch): 116s + Train loss: 0.831307 (1278.4s) ETA: 2050min + Val loss: 0.866086 [t_0.0-0.2=1.0888 t_0.2-0.4=1.0108 t_0.4-0.6=0.8285 t_0.6-0.8=0.7020 t_0.8-1.0=0.6269] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 213/299 (71% done) --- + [Epoch 213][10/66] loss=0.911416 avg=0.846628 VRAM=39.0GiB | 71.1% done | ETA(epoch): 1080s + [Epoch 213][20/66] loss=0.928861 avg=0.874828 VRAM=38.9GiB | 71.1% done | ETA(epoch): 887s + [Epoch 213][30/66] loss=0.873882 avg=0.875502 VRAM=39.0GiB | 71.2% done | ETA(epoch): 695s + [Epoch 213][40/66] loss=0.847757 avg=0.877227 VRAM=38.9GiB | 71.2% done | ETA(epoch): 502s + [Epoch 213][50/66] loss=0.835960 avg=0.862710 VRAM=39.0GiB | 71.3% done | ETA(epoch): 309s + [Epoch 213][60/66] loss=0.760175 avg=0.856284 VRAM=38.9GiB | 71.3% done | ETA(epoch): 116s + Train loss: 0.853871 (1274.1s) ETA: 2026min + Val loss: 0.841435 [t_0.0-0.2=1.0488 t_0.2-0.4=0.9993 t_0.4-0.6=0.8389 t_0.6-0.8=0.7081 t_0.8-1.0=0.6306] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 214/299 (71% done) --- + [Epoch 214][10/66] loss=0.881761 avg=0.865350 VRAM=39.0GiB | 71.4% done | ETA(epoch): 1082s + [Epoch 214][20/66] loss=0.789822 avg=0.868038 VRAM=38.9GiB | 71.4% done | ETA(epoch): 888s + [Epoch 214][30/66] loss=1.022004 avg=0.874530 VRAM=39.0GiB | 71.5% done | ETA(epoch): 695s + [Epoch 214][40/66] loss=0.937246 avg=0.861660 VRAM=38.9GiB | 71.5% done | ETA(epoch): 502s + [Epoch 214][50/66] loss=0.950873 avg=0.853876 VRAM=39.0GiB | 71.6% done | ETA(epoch): 309s + [Epoch 214][60/66] loss=0.838566 avg=0.851733 VRAM=38.9GiB | 71.6% done | ETA(epoch): 116s + Train loss: 0.848715 (1275.7s) ETA: 2002min + Val loss: 0.846319 [t_0.0-0.2=1.0810 t_0.2-0.4=0.9736 t_0.4-0.6=0.8407 t_0.6-0.8=0.6856 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 215/299 (72% done) --- + [Epoch 215][10/66] loss=0.745124 avg=0.803510 VRAM=39.0GiB | 71.7% done | ETA(epoch): 1081s + [Epoch 215][20/66] loss=0.681255 avg=0.808093 VRAM=38.9GiB | 71.8% done | ETA(epoch): 888s + [Epoch 215][30/66] loss=0.899735 avg=0.828994 VRAM=39.0GiB | 71.8% done | ETA(epoch): 695s + [Epoch 215][40/66] loss=0.825790 avg=0.827714 VRAM=38.9GiB | 71.9% done | ETA(epoch): 502s + [Epoch 215][50/66] loss=0.648023 avg=0.822666 VRAM=39.0GiB | 71.9% done | ETA(epoch): 309s + [Epoch 215][60/66] loss=0.924950 avg=0.831304 VRAM=38.9GiB | 72.0% done | ETA(epoch): 116s + Train loss: 0.832435 (1274.0s) ETA: 1979min + Val loss: 0.851459 [t_0.0-0.2=1.0764 t_0.2-0.4=1.0004 t_0.4-0.6=0.8163 t_0.6-0.8=0.7147 t_0.8-1.0=0.6331] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 216/299 (72% done) --- + [Epoch 216][10/66] loss=0.869562 avg=0.864685 VRAM=39.0GiB | 72.1% done | ETA(epoch): 1089s + [Epoch 216][20/66] loss=0.875647 avg=0.860061 VRAM=38.9GiB | 72.1% done | ETA(epoch): 892s + [Epoch 216][30/66] loss=0.717063 avg=0.865004 VRAM=39.0GiB | 72.2% done | ETA(epoch): 697s + [Epoch 216][40/66] loss=0.773660 avg=0.862189 VRAM=38.9GiB | 72.2% done | ETA(epoch): 503s + [Epoch 216][50/66] loss=0.978411 avg=0.865358 VRAM=39.0GiB | 72.3% done | ETA(epoch): 309s + [Epoch 216][60/66] loss=0.862175 avg=0.863317 VRAM=38.9GiB | 72.3% done | ETA(epoch): 116s + Train loss: 0.863694 (1277.1s) ETA: 1955min + Val loss: 0.860573 [t_0.0-0.2=1.0801 t_0.2-0.4=1.0063 t_0.4-0.6=0.8642 t_0.6-0.8=0.6920 t_0.8-1.0=0.6216] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 21.2/188.5 GiB (11.2%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 217/299 (72% done) --- + [Epoch 217][10/66] loss=0.808441 avg=0.835084 VRAM=39.0GiB | 72.4% done | ETA(epoch): 1081s + [Epoch 217][20/66] loss=0.797452 avg=0.839052 VRAM=38.9GiB | 72.4% done | ETA(epoch): 888s + [Epoch 217][30/66] loss=0.787767 avg=0.825253 VRAM=39.0GiB | 72.5% done | ETA(epoch): 695s + [Epoch 217][40/66] loss=0.782338 avg=0.836117 VRAM=38.9GiB | 72.5% done | ETA(epoch): 502s + [Epoch 217][50/66] loss=0.966676 avg=0.847209 VRAM=39.0GiB | 72.6% done | ETA(epoch): 309s + [Epoch 217][60/66] loss=0.921152 avg=0.854857 VRAM=38.9GiB | 72.6% done | ETA(epoch): 116s + Train loss: 0.857898 (1275.7s) ETA: 1931min + Val loss: 0.872298 [t_0.0-0.2=1.0773 t_0.2-0.4=1.0038 t_0.4-0.6=0.8589 t_0.6-0.8=0.6991 t_0.8-1.0=0.6407] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 218/299 (73% done) --- + [Epoch 218][10/66] loss=0.772325 avg=0.858991 VRAM=39.0GiB | 72.7% done | ETA(epoch): 1082s + [Epoch 218][20/66] loss=0.880954 avg=0.851178 VRAM=38.9GiB | 72.8% done | ETA(epoch): 889s + [Epoch 218][30/66] loss=0.886238 avg=0.845645 VRAM=39.0GiB | 72.8% done | ETA(epoch): 696s + [Epoch 218][40/66] loss=0.922597 avg=0.853623 VRAM=38.9GiB | 72.9% done | ETA(epoch): 503s + [Epoch 218][50/66] loss=0.940934 avg=0.848730 VRAM=39.0GiB | 72.9% done | ETA(epoch): 309s + [Epoch 218][60/66] loss=1.009388 avg=0.845082 VRAM=38.9GiB | 73.0% done | ETA(epoch): 116s + Train loss: 0.845679 (1275.4s) ETA: 1907min + Val loss: 0.865184 [t_0.0-0.2=1.0685 t_0.2-0.4=0.9948 t_0.4-0.6=0.8221 t_0.6-0.8=0.7241 t_0.8-1.0=0.6386] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 219/299 (73% done) --- + [Epoch 219][10/66] loss=1.003951 avg=0.832524 VRAM=39.0GiB | 73.1% done | ETA(epoch): 1087s + [Epoch 219][20/66] loss=0.801504 avg=0.839527 VRAM=38.9GiB | 73.1% done | ETA(epoch): 892s + [Epoch 219][30/66] loss=0.918993 avg=0.856084 VRAM=39.0GiB | 73.2% done | ETA(epoch): 698s + [Epoch 219][40/66] loss=0.851208 avg=0.841985 VRAM=38.9GiB | 73.2% done | ETA(epoch): 503s + [Epoch 219][50/66] loss=0.898009 avg=0.838569 VRAM=39.0GiB | 73.3% done | ETA(epoch): 309s + [Epoch 219][60/66] loss=0.932742 avg=0.840602 VRAM=38.9GiB | 73.3% done | ETA(epoch): 116s + Train loss: 0.845442 (1276.4s) ETA: 1883min + Val loss: 0.872497 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9692 t_0.4-0.6=0.8372 t_0.6-0.8=0.7128 t_0.8-1.0=0.6339] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 220/299 (73% done) --- + [Epoch 220][10/66] loss=0.723686 avg=0.842081 VRAM=39.0GiB | 73.4% done | ETA(epoch): 1082s + [Epoch 220][20/66] loss=0.771951 avg=0.819743 VRAM=38.9GiB | 73.4% done | ETA(epoch): 889s + [Epoch 220][30/66] loss=0.771358 avg=0.827158 VRAM=39.0GiB | 73.5% done | ETA(epoch): 695s + [Epoch 220][40/66] loss=0.769480 avg=0.831033 VRAM=38.9GiB | 73.5% done | ETA(epoch): 503s + [Epoch 220][50/66] loss=0.734043 avg=0.830987 VRAM=39.0GiB | 73.6% done | ETA(epoch): 309s + [Epoch 220][60/66] loss=0.920499 avg=0.833454 VRAM=38.9GiB | 73.6% done | ETA(epoch): 116s + Train loss: 0.828187 (1275.6s) ETA: 1860min + Val loss: 0.829296 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9851 t_0.4-0.6=0.8055 t_0.6-0.8=0.7023 t_0.8-1.0=0.6273] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 221/299 (74% done) --- + [Epoch 221][10/66] loss=0.919629 avg=0.896628 VRAM=39.0GiB | 73.7% done | ETA(epoch): 1082s + [Epoch 221][20/66] loss=0.892355 avg=0.873666 VRAM=38.9GiB | 73.8% done | ETA(epoch): 889s + [Epoch 221][30/66] loss=0.805991 avg=0.864363 VRAM=39.0GiB | 73.8% done | ETA(epoch): 696s + [Epoch 221][40/66] loss=0.883394 avg=0.871547 VRAM=38.9GiB | 73.9% done | ETA(epoch): 502s + [Epoch 221][50/66] loss=0.860669 avg=0.869174 VRAM=39.0GiB | 73.9% done | ETA(epoch): 309s + [Epoch 221][60/66] loss=0.786407 avg=0.868610 VRAM=38.9GiB | 74.0% done | ETA(epoch): 116s + Train loss: 0.872056 (1276.4s) ETA: 1836min + Val loss: 0.828673 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9912 t_0.4-0.6=0.8498 t_0.6-0.8=0.7051 t_0.8-1.0=0.6297] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 222/299 (74% done) --- + [Epoch 222][10/66] loss=0.918862 avg=0.869337 VRAM=39.0GiB | 74.1% done | ETA(epoch): 1082s + [Epoch 222][20/66] loss=0.961874 avg=0.868281 VRAM=38.9GiB | 74.1% done | ETA(epoch): 889s + [Epoch 222][30/66] loss=0.776085 avg=0.860991 VRAM=39.0GiB | 74.2% done | ETA(epoch): 697s + [Epoch 222][40/66] loss=0.831447 avg=0.856191 VRAM=38.9GiB | 74.2% done | ETA(epoch): 504s + [Epoch 222][50/66] loss=0.895364 avg=0.864497 VRAM=39.0GiB | 74.3% done | ETA(epoch): 310s + [Epoch 222][60/66] loss=0.777499 avg=0.860833 VRAM=38.9GiB | 74.3% done | ETA(epoch): 116s + Train loss: 0.861433 (1278.4s) ETA: 1812min + Val loss: 0.845574 [t_0.0-0.2=1.0854 t_0.2-0.4=0.9927 t_0.4-0.6=0.8516 t_0.6-0.8=0.7059 t_0.8-1.0=0.6229] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 223/299 (74% done) --- + [Epoch 223][10/66] loss=0.839452 avg=0.875473 VRAM=39.0GiB | 74.4% done | ETA(epoch): 1118s + [Epoch 223][20/66] loss=0.936276 avg=0.880623 VRAM=38.9GiB | 74.4% done | ETA(epoch): 903s + [Epoch 223][30/66] loss=0.828201 avg=0.869977 VRAM=39.0GiB | 74.5% done | ETA(epoch): 703s + [Epoch 223][40/66] loss=0.774984 avg=0.872279 VRAM=38.9GiB | 74.5% done | ETA(epoch): 506s + [Epoch 223][50/66] loss=0.828376 avg=0.865750 VRAM=39.0GiB | 74.6% done | ETA(epoch): 311s + [Epoch 223][60/66] loss=0.709563 avg=0.862178 VRAM=38.9GiB | 74.6% done | ETA(epoch): 117s + Train loss: 0.860474 (1281.7s) ETA: 1788min + Val loss: 0.874050 [t_0.0-0.2=1.0818 t_0.2-0.4=1.0285 t_0.4-0.6=0.7993 t_0.6-0.8=0.7142 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 224/299 (75% done) --- + [Epoch 224][10/66] loss=0.792611 avg=0.829258 VRAM=39.0GiB | 74.7% done | ETA(epoch): 1080s + [Epoch 224][20/66] loss=0.926365 avg=0.845263 VRAM=38.9GiB | 74.8% done | ETA(epoch): 887s + [Epoch 224][30/66] loss=0.770789 avg=0.850823 VRAM=39.0GiB | 74.8% done | ETA(epoch): 695s + [Epoch 224][40/66] loss=0.782291 avg=0.851185 VRAM=38.9GiB | 74.9% done | ETA(epoch): 502s + [Epoch 224][50/66] loss=1.021815 avg=0.852928 VRAM=39.0GiB | 74.9% done | ETA(epoch): 309s + [Epoch 224][60/66] loss=0.919769 avg=0.855881 VRAM=38.9GiB | 75.0% done | ETA(epoch): 116s + Train loss: 0.852629 (1274.5s) ETA: 1764min + Val loss: 0.818069 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9757 t_0.4-0.6=0.8775 t_0.6-0.8=0.7063 t_0.8-1.0=0.6352] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 225/299 (75% done) --- + [Epoch 225][10/66] loss=0.889357 avg=0.839756 VRAM=39.0GiB | 75.1% done | ETA(epoch): 1081s + [Epoch 225][20/66] loss=1.016511 avg=0.861722 VRAM=38.9GiB | 75.1% done | ETA(epoch): 888s + [Epoch 225][30/66] loss=0.804142 avg=0.856651 VRAM=39.0GiB | 75.2% done | ETA(epoch): 695s + [Epoch 225][40/66] loss=0.819382 avg=0.848974 VRAM=38.9GiB | 75.2% done | ETA(epoch): 502s + [Epoch 225][50/66] loss=0.835955 avg=0.844448 VRAM=39.0GiB | 75.3% done | ETA(epoch): 309s + [Epoch 225][60/66] loss=0.732308 avg=0.836693 VRAM=38.9GiB | 75.3% done | ETA(epoch): 116s + Train loss: 0.834218 (1278.4s) ETA: 1741min + Val loss: 0.806411 [t_0.0-0.2=1.0701 t_0.2-0.4=0.9979 t_0.4-0.6=0.8729 t_0.6-0.8=0.6989 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 226/299 (75% done) --- + [Epoch 226][10/66] loss=0.865043 avg=0.844315 VRAM=39.0GiB | 75.4% done | ETA(epoch): 1081s + [Epoch 226][20/66] loss=0.880251 avg=0.849439 VRAM=38.9GiB | 75.4% done | ETA(epoch): 888s + [Epoch 226][30/66] loss=0.807361 avg=0.854641 VRAM=39.0GiB | 75.5% done | ETA(epoch): 695s + [Epoch 226][40/66] loss=0.891194 avg=0.862192 VRAM=38.9GiB | 75.5% done | ETA(epoch): 502s + [Epoch 226][50/66] loss=0.775501 avg=0.857981 VRAM=39.0GiB | 75.6% done | ETA(epoch): 309s + [Epoch 226][60/66] loss=0.876712 avg=0.848998 VRAM=38.9GiB | 75.6% done | ETA(epoch): 116s + Train loss: 0.848993 (1274.9s) ETA: 1717min + Val loss: 0.835050 [t_0.0-0.2=1.0893 t_0.2-0.4=0.9933 t_0.4-0.6=0.8287 t_0.6-0.8=0.7111 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 227/299 (76% done) --- + [Epoch 227][10/66] loss=0.941945 avg=0.868565 VRAM=39.0GiB | 75.7% done | ETA(epoch): 1082s + [Epoch 227][20/66] loss=0.702319 avg=0.854430 VRAM=38.9GiB | 75.8% done | ETA(epoch): 888s + [Epoch 227][30/66] loss=0.839240 avg=0.857677 VRAM=39.0GiB | 75.8% done | ETA(epoch): 695s + [Epoch 227][40/66] loss=0.858548 avg=0.857329 VRAM=38.9GiB | 75.9% done | ETA(epoch): 506s + [Epoch 227][50/66] loss=1.029869 avg=0.855483 VRAM=39.0GiB | 75.9% done | ETA(epoch): 311s + [Epoch 227][60/66] loss=0.838456 avg=0.853204 VRAM=38.9GiB | 76.0% done | ETA(epoch): 116s + Train loss: 0.856062 (1280.3s) ETA: 1693min + Val loss: 0.842810 [t_0.0-0.2=1.0777 t_0.2-0.4=1.0056 t_0.4-0.6=0.8130 t_0.6-0.8=0.7027 t_0.8-1.0=0.6070] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 228/299 (76% done) --- + [Epoch 228][10/66] loss=0.863009 avg=0.879867 VRAM=39.0GiB | 76.1% done | ETA(epoch): 1080s + [Epoch 228][20/66] loss=0.707877 avg=0.844339 VRAM=38.9GiB | 76.1% done | ETA(epoch): 888s + [Epoch 228][30/66] loss=0.845196 avg=0.843113 VRAM=39.0GiB | 76.2% done | ETA(epoch): 695s + [Epoch 228][40/66] loss=0.748888 avg=0.850904 VRAM=38.9GiB | 76.2% done | ETA(epoch): 502s + [Epoch 228][50/66] loss=0.755990 avg=0.844933 VRAM=39.0GiB | 76.3% done | ETA(epoch): 309s + [Epoch 228][60/66] loss=0.723485 avg=0.851524 VRAM=38.9GiB | 76.3% done | ETA(epoch): 116s + Train loss: 0.848250 (1274.7s) ETA: 1670min + Val loss: 0.850758 [t_0.0-0.2=1.0721 t_0.2-0.4=0.9666 t_0.4-0.6=0.8284 t_0.6-0.8=0.6982 t_0.8-1.0=0.6294] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 229/299 (76% done) --- + [Epoch 229][10/66] loss=0.881896 avg=0.840716 VRAM=39.0GiB | 76.4% done | ETA(epoch): 1083s + [Epoch 229][20/66] loss=0.936444 avg=0.862947 VRAM=38.9GiB | 76.4% done | ETA(epoch): 894s + [Epoch 229][30/66] loss=0.745644 avg=0.865879 VRAM=39.0GiB | 76.5% done | ETA(epoch): 699s + [Epoch 229][40/66] loss=0.874529 avg=0.857038 VRAM=38.9GiB | 76.5% done | ETA(epoch): 505s + [Epoch 229][50/66] loss=0.910663 avg=0.856782 VRAM=39.0GiB | 76.6% done | ETA(epoch): 311s + [Epoch 229][60/66] loss=0.766179 avg=0.855845 VRAM=38.9GiB | 76.6% done | ETA(epoch): 116s + Train loss: 0.855364 (1279.7s) ETA: 1646min + Val loss: 0.826913 [t_0.0-0.2=1.0595 t_0.2-0.4=0.9881 t_0.4-0.6=0.8197 t_0.6-0.8=0.7193 t_0.8-1.0=0.6348] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 230/299 (77% done) --- + [Epoch 230][10/66] loss=0.756133 avg=0.852771 VRAM=39.0GiB | 76.7% done | ETA(epoch): 1082s + [Epoch 230][20/66] loss=0.852049 avg=0.834614 VRAM=38.9GiB | 76.8% done | ETA(epoch): 889s + [Epoch 230][30/66] loss=0.821127 avg=0.842789 VRAM=39.0GiB | 76.8% done | ETA(epoch): 696s + [Epoch 230][40/66] loss=0.843610 avg=0.849426 VRAM=38.9GiB | 76.9% done | ETA(epoch): 502s + [Epoch 230][50/66] loss=0.710058 avg=0.847667 VRAM=39.0GiB | 76.9% done | ETA(epoch): 309s + [Epoch 230][60/66] loss=0.854952 avg=0.851762 VRAM=38.9GiB | 77.0% done | ETA(epoch): 116s + Train loss: 0.843336 (1274.8s) ETA: 1622min + Val loss: 0.865716 [t_0.0-0.2=1.0857 t_0.2-0.4=0.9851 t_0.4-0.6=0.8661 t_0.6-0.8=0.7131 t_0.8-1.0=0.6336] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 231/299 (77% done) --- + [Epoch 231][10/66] loss=0.959908 avg=0.885052 VRAM=39.0GiB | 77.1% done | ETA(epoch): 1088s + [Epoch 231][20/66] loss=0.657408 avg=0.887838 VRAM=38.9GiB | 77.1% done | ETA(epoch): 893s + [Epoch 231][30/66] loss=0.820304 avg=0.868248 VRAM=39.0GiB | 77.2% done | ETA(epoch): 698s + [Epoch 231][40/66] loss=0.902919 avg=0.859240 VRAM=38.9GiB | 77.2% done | ETA(epoch): 505s + [Epoch 231][50/66] loss=0.926594 avg=0.854260 VRAM=39.0GiB | 77.3% done | ETA(epoch): 310s + [Epoch 231][60/66] loss=0.934425 avg=0.859741 VRAM=38.9GiB | 77.3% done | ETA(epoch): 116s + Train loss: 0.857541 (1278.9s) ETA: 1599min + Val loss: 0.850699 [t_0.0-0.2=1.0573 t_0.2-0.4=0.9792 t_0.4-0.6=0.8450 t_0.6-0.8=0.7092 t_0.8-1.0=0.6434] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 232/299 (77% done) --- + [Epoch 232][10/66] loss=0.962589 avg=0.902649 VRAM=39.0GiB | 77.4% done | ETA(epoch): 1083s + [Epoch 232][20/66] loss=0.796529 avg=0.873123 VRAM=38.9GiB | 77.4% done | ETA(epoch): 892s + [Epoch 232][30/66] loss=0.764286 avg=0.864658 VRAM=39.0GiB | 77.5% done | ETA(epoch): 699s + [Epoch 232][40/66] loss=0.942598 avg=0.855174 VRAM=38.9GiB | 77.5% done | ETA(epoch): 504s + [Epoch 232][50/66] loss=0.944546 avg=0.850492 VRAM=39.0GiB | 77.6% done | ETA(epoch): 310s + [Epoch 232][60/66] loss=0.910931 avg=0.853706 VRAM=38.9GiB | 77.6% done | ETA(epoch): 116s + Train loss: 0.853833 (1279.7s) ETA: 1575min + Val loss: 0.838661 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9540 t_0.4-0.6=0.8318 t_0.6-0.8=0.7136 t_0.8-1.0=0.6289] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 233/299 (78% done) --- + [Epoch 233][10/66] loss=0.858544 avg=0.817007 VRAM=39.0GiB | 77.7% done | ETA(epoch): 1080s + [Epoch 233][20/66] loss=0.936874 avg=0.823225 VRAM=38.9GiB | 77.8% done | ETA(epoch): 888s + [Epoch 233][30/66] loss=0.878407 avg=0.839368 VRAM=39.0GiB | 77.8% done | ETA(epoch): 695s + [Epoch 233][40/66] loss=0.882829 avg=0.838924 VRAM=38.9GiB | 77.9% done | ETA(epoch): 502s + [Epoch 233][50/66] loss=0.973128 avg=0.839265 VRAM=39.0GiB | 77.9% done | ETA(epoch): 310s + [Epoch 233][60/66] loss=0.705833 avg=0.841474 VRAM=38.9GiB | 78.0% done | ETA(epoch): 116s + Train loss: 0.842954 (1276.8s) ETA: 1551min + Val loss: 0.878032 [t_0.0-0.2=1.0915 t_0.2-0.4=0.9947 t_0.4-0.6=0.8647 t_0.6-0.8=0.7147 t_0.8-1.0=0.6390] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 234/299 (78% done) --- + [Epoch 234][10/66] loss=0.900101 avg=0.827825 VRAM=39.0GiB | 78.1% done | ETA(epoch): 1081s + [Epoch 234][20/66] loss=0.856473 avg=0.839634 VRAM=38.9GiB | 78.1% done | ETA(epoch): 888s + [Epoch 234][30/66] loss=0.989856 avg=0.851706 VRAM=39.0GiB | 78.2% done | ETA(epoch): 696s + [Epoch 234][40/66] loss=0.836870 avg=0.852145 VRAM=38.9GiB | 78.2% done | ETA(epoch): 503s + [Epoch 234][50/66] loss=0.817205 avg=0.843763 VRAM=39.0GiB | 78.3% done | ETA(epoch): 309s + [Epoch 234][60/66] loss=0.861484 avg=0.843877 VRAM=38.9GiB | 78.3% done | ETA(epoch): 116s + Train loss: 0.833880 (1276.3s) ETA: 1527min + Val loss: 0.870401 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9969 t_0.4-0.6=0.8373 t_0.6-0.8=0.7293 t_0.8-1.0=0.6375] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 235/299 (78% done) --- + [Epoch 235][10/66] loss=0.893470 avg=0.817293 VRAM=39.0GiB | 78.4% done | ETA(epoch): 1085s + [Epoch 235][20/66] loss=0.846647 avg=0.839623 VRAM=38.9GiB | 78.4% done | ETA(epoch): 891s + [Epoch 235][30/66] loss=0.782613 avg=0.837947 VRAM=39.0GiB | 78.5% done | ETA(epoch): 699s + [Epoch 235][40/66] loss=0.815220 avg=0.831001 VRAM=38.9GiB | 78.5% done | ETA(epoch): 504s + [Epoch 235][50/66] loss=0.885999 avg=0.825904 VRAM=39.0GiB | 78.6% done | ETA(epoch): 310s + [Epoch 235][60/66] loss=0.850357 avg=0.827133 VRAM=38.9GiB | 78.6% done | ETA(epoch): 116s + Train loss: 0.827432 (1278.3s) ETA: 1504min + Val loss: 0.831999 [t_0.0-0.2=1.0572 t_0.2-0.4=1.0100 t_0.4-0.6=0.8301 t_0.6-0.8=0.7212 t_0.8-1.0=0.6324] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 236/299 (79% done) --- + [Epoch 236][10/66] loss=0.765178 avg=0.839968 VRAM=39.0GiB | 78.7% done | ETA(epoch): 1080s + [Epoch 236][20/66] loss=1.011911 avg=0.836582 VRAM=38.9GiB | 78.8% done | ETA(epoch): 897s + [Epoch 236][30/66] loss=0.723136 avg=0.839756 VRAM=39.0GiB | 78.8% done | ETA(epoch): 700s + [Epoch 236][40/66] loss=0.847418 avg=0.839921 VRAM=38.9GiB | 78.9% done | ETA(epoch): 505s + [Epoch 236][50/66] loss=0.788390 avg=0.832114 VRAM=39.0GiB | 78.9% done | ETA(epoch): 310s + [Epoch 236][60/66] loss=0.775502 avg=0.832119 VRAM=38.9GiB | 79.0% done | ETA(epoch): 116s + Train loss: 0.833368 (1278.7s) ETA: 1480min + Val loss: 0.825523 [t_0.0-0.2=1.0625 t_0.2-0.4=0.9906 t_0.4-0.6=0.8422 t_0.6-0.8=0.6909 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 237/299 (79% done) --- + [Epoch 237][10/66] loss=0.821557 avg=0.865137 VRAM=39.0GiB | 79.1% done | ETA(epoch): 1094s + [Epoch 237][20/66] loss=0.782219 avg=0.856343 VRAM=38.9GiB | 79.1% done | ETA(epoch): 893s + [Epoch 237][30/66] loss=0.739681 avg=0.853230 VRAM=39.0GiB | 79.2% done | ETA(epoch): 698s + [Epoch 237][40/66] loss=0.865731 avg=0.847441 VRAM=38.9GiB | 79.2% done | ETA(epoch): 504s + [Epoch 237][50/66] loss=0.843772 avg=0.851584 VRAM=39.0GiB | 79.3% done | ETA(epoch): 310s + [Epoch 237][60/66] loss=0.924889 avg=0.851824 VRAM=38.9GiB | 79.3% done | ETA(epoch): 116s + Train loss: 0.853451 (1277.2s) ETA: 1456min + Val loss: 0.859659 [t_0.0-0.2=1.0622 t_0.2-0.4=1.0295 t_0.4-0.6=0.8451 t_0.6-0.8=0.7088 t_0.8-1.0=0.6494] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 21.8/188.5 GiB (11.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 238/299 (79% done) --- + [Epoch 238][10/66] loss=0.983658 avg=0.857939 VRAM=39.0GiB | 79.4% done | ETA(epoch): 1081s + [Epoch 238][20/66] loss=0.797747 avg=0.844310 VRAM=38.9GiB | 79.4% done | ETA(epoch): 889s + [Epoch 238][30/66] loss=0.904302 avg=0.833793 VRAM=39.0GiB | 79.5% done | ETA(epoch): 696s + [Epoch 238][40/66] loss=0.792113 avg=0.830039 VRAM=38.9GiB | 79.5% done | ETA(epoch): 503s + [Epoch 238][50/66] loss=0.771286 avg=0.836304 VRAM=39.0GiB | 79.6% done | ETA(epoch): 310s + [Epoch 238][60/66] loss=0.925822 avg=0.837822 VRAM=38.9GiB | 79.6% done | ETA(epoch): 116s + Train loss: 0.839058 (1282.2s) ETA: 1433min + Val loss: 0.855041 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9709 t_0.4-0.6=0.8325 t_0.6-0.8=0.7379 t_0.8-1.0=0.6441] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 239/299 (80% done) --- + [Epoch 239][10/66] loss=0.859600 avg=0.842642 VRAM=39.0GiB | 79.7% done | ETA(epoch): 1083s + [Epoch 239][20/66] loss=0.905434 avg=0.821176 VRAM=38.9GiB | 79.8% done | ETA(epoch): 891s + [Epoch 239][30/66] loss=0.842128 avg=0.834941 VRAM=39.0GiB | 79.8% done | ETA(epoch): 697s + [Epoch 239][40/66] loss=0.847623 avg=0.851201 VRAM=38.9GiB | 79.9% done | ETA(epoch): 505s + [Epoch 239][50/66] loss=0.875590 avg=0.860538 VRAM=39.0GiB | 79.9% done | ETA(epoch): 310s + [Epoch 239][60/66] loss=0.836944 avg=0.859111 VRAM=38.9GiB | 80.0% done | ETA(epoch): 117s + [MilestoneVis] train_0 step 15839 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 15839 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 15839 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 15839 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 15839 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 15839 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 15839 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 15839 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 15839 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 15839 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.857786 (2326.4s) ETA: 1414min + Val loss: 0.823518 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9732 t_0.4-0.6=0.8055 t_0.6-0.8=0.7183 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 22.8/188.5 GiB (12.1%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 240/299 (80% done) --- + [Epoch 240][10/66] loss=0.797754 avg=0.877667 VRAM=39.0GiB | 80.1% done | ETA(epoch): 1084s + [Epoch 240][20/66] loss=0.899485 avg=0.878510 VRAM=38.9GiB | 80.1% done | ETA(epoch): 890s + [Epoch 240][30/66] loss=0.944757 avg=0.866565 VRAM=39.0GiB | 80.2% done | ETA(epoch): 696s + [Epoch 240][40/66] loss=0.747363 avg=0.866137 VRAM=38.9GiB | 80.2% done | ETA(epoch): 503s + [Epoch 240][50/66] loss=0.823262 avg=0.871975 VRAM=39.0GiB | 80.3% done | ETA(epoch): 310s + [Epoch 240][60/66] loss=0.894123 avg=0.870178 VRAM=38.9GiB | 80.3% done | ETA(epoch): 116s + Train loss: 0.868434 (1280.4s) ETA: 1390min + Val loss: 0.855956 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9956 t_0.4-0.6=0.8274 t_0.6-0.8=0.6949 t_0.8-1.0=0.6169] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 22.8/188.5 GiB (12.1%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 241/299 (80% done) --- + [Epoch 241][10/66] loss=0.915019 avg=0.837992 VRAM=39.0GiB | 80.4% done | ETA(epoch): 1090s + [Epoch 241][20/66] loss=0.992901 avg=0.849399 VRAM=38.9GiB | 80.4% done | ETA(epoch): 894s + [Epoch 241][30/66] loss=0.842295 avg=0.847166 VRAM=39.0GiB | 80.5% done | ETA(epoch): 700s + [Epoch 241][40/66] loss=0.943380 avg=0.851519 VRAM=38.9GiB | 80.5% done | ETA(epoch): 505s + [Epoch 241][50/66] loss=0.856736 avg=0.850000 VRAM=39.0GiB | 80.6% done | ETA(epoch): 311s + [Epoch 241][60/66] loss=0.849670 avg=0.849838 VRAM=38.9GiB | 80.6% done | ETA(epoch): 117s + Train loss: 0.850535 (1283.0s) ETA: 1366min + Val loss: 0.878638 [t_0.0-0.2=1.0737 t_0.2-0.4=1.0243 t_0.4-0.6=0.8569 t_0.6-0.8=0.7130 t_0.8-1.0=0.6344] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 242/299 (81% done) --- + [Epoch 242][10/66] loss=0.814477 avg=0.841635 VRAM=39.0GiB | 80.7% done | ETA(epoch): 1088s + [Epoch 242][20/66] loss=0.896927 avg=0.852489 VRAM=38.9GiB | 80.8% done | ETA(epoch): 893s + [Epoch 242][30/66] loss=0.754072 avg=0.846480 VRAM=39.0GiB | 80.8% done | ETA(epoch): 699s + [Epoch 242][40/66] loss=1.003915 avg=0.856730 VRAM=38.9GiB | 80.9% done | ETA(epoch): 507s + [Epoch 242][50/66] loss=0.890684 avg=0.850250 VRAM=39.0GiB | 80.9% done | ETA(epoch): 311s + [Epoch 242][60/66] loss=0.766151 avg=0.851631 VRAM=38.9GiB | 81.0% done | ETA(epoch): 117s + Train loss: 0.848391 (1282.1s) ETA: 1343min + Val loss: 0.871269 [t_0.0-0.2=1.0709 t_0.2-0.4=0.9892 t_0.4-0.6=0.8559 t_0.6-0.8=0.6996 t_0.8-1.0=0.6320] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 243/299 (81% done) --- + [Epoch 243][10/66] loss=0.847090 avg=0.826220 VRAM=39.0GiB | 81.1% done | ETA(epoch): 1081s + [Epoch 243][20/66] loss=0.954479 avg=0.859585 VRAM=38.9GiB | 81.1% done | ETA(epoch): 889s + [Epoch 243][30/66] loss=0.885618 avg=0.862955 VRAM=39.0GiB | 81.2% done | ETA(epoch): 699s + [Epoch 243][40/66] loss=0.802761 avg=0.858583 VRAM=38.9GiB | 81.2% done | ETA(epoch): 505s + [Epoch 243][50/66] loss=0.680469 avg=0.857047 VRAM=39.0GiB | 81.3% done | ETA(epoch): 311s + [Epoch 243][60/66] loss=1.026432 avg=0.858158 VRAM=38.9GiB | 81.3% done | ETA(epoch): 117s + Train loss: 0.861471 (1281.6s) ETA: 1319min + Val loss: 0.853115 [t_0.0-0.2=1.0591 t_0.2-0.4=0.9716 t_0.4-0.6=0.8647 t_0.6-0.8=0.7004 t_0.8-1.0=0.6379] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 21.8/188.5 GiB (11.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 244/299 (81% done) --- + [Epoch 244][10/66] loss=0.953938 avg=0.859333 VRAM=39.0GiB | 81.4% done | ETA(epoch): 1080s + [Epoch 244][20/66] loss=0.953856 avg=0.845267 VRAM=38.9GiB | 81.4% done | ETA(epoch): 899s + [Epoch 244][30/66] loss=0.825480 avg=0.844068 VRAM=39.0GiB | 81.5% done | ETA(epoch): 701s + [Epoch 244][40/66] loss=0.882001 avg=0.860545 VRAM=38.9GiB | 81.5% done | ETA(epoch): 507s + [Epoch 244][50/66] loss=0.756563 avg=0.851944 VRAM=39.0GiB | 81.6% done | ETA(epoch): 312s + [Epoch 244][60/66] loss=0.868798 avg=0.856747 VRAM=38.9GiB | 81.6% done | ETA(epoch): 117s + Train loss: 0.860510 (1286.0s) ETA: 1295min + Val loss: 0.840815 [t_0.0-0.2=1.0546 t_0.2-0.4=0.9866 t_0.4-0.6=0.8600 t_0.6-0.8=0.7053 t_0.8-1.0=0.6310] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 245/299 (82% done) --- + [Epoch 245][10/66] loss=0.952995 avg=0.830608 VRAM=39.0GiB | 81.7% done | ETA(epoch): 1083s + [Epoch 245][20/66] loss=0.951697 avg=0.845961 VRAM=38.9GiB | 81.8% done | ETA(epoch): 890s + [Epoch 245][30/66] loss=0.778752 avg=0.841469 VRAM=39.0GiB | 81.8% done | ETA(epoch): 696s + [Epoch 245][40/66] loss=0.967166 avg=0.848971 VRAM=38.9GiB | 81.9% done | ETA(epoch): 503s + [Epoch 245][50/66] loss=0.964805 avg=0.847364 VRAM=39.0GiB | 81.9% done | ETA(epoch): 310s + [Epoch 245][60/66] loss=0.769183 avg=0.849671 VRAM=38.9GiB | 82.0% done | ETA(epoch): 116s + Train loss: 0.850318 (1277.0s) ETA: 1271min + Val loss: 0.846267 [t_0.0-0.2=1.0622 t_0.2-0.4=0.9399 t_0.4-0.6=0.8515 t_0.6-0.8=0.7166 t_0.8-1.0=0.6241] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0245 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 246/299 (82% done) --- + [Epoch 246][10/66] loss=0.798297 avg=0.879225 VRAM=39.0GiB | 82.1% done | ETA(epoch): 1084s + [Epoch 246][20/66] loss=0.935539 avg=0.875326 VRAM=38.9GiB | 82.1% done | ETA(epoch): 893s + [Epoch 246][30/66] loss=0.917953 avg=0.871104 VRAM=39.0GiB | 82.2% done | ETA(epoch): 698s + [Epoch 246][40/66] loss=0.763921 avg=0.866748 VRAM=38.9GiB | 82.2% done | ETA(epoch): 504s + [Epoch 246][50/66] loss=0.943305 avg=0.869254 VRAM=39.0GiB | 82.3% done | ETA(epoch): 310s + [Epoch 246][60/66] loss=0.818572 avg=0.865074 VRAM=38.9GiB | 82.3% done | ETA(epoch): 116s + Train loss: 0.860445 (1277.2s) ETA: 1248min + Val loss: 0.896722 [t_0.0-0.2=1.0760 t_0.2-0.4=1.0188 t_0.4-0.6=0.8480 t_0.6-0.8=0.7050 t_0.8-1.0=0.6337] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 247/299 (82% done) --- + [Epoch 247][10/66] loss=0.697656 avg=0.803025 VRAM=39.0GiB | 82.4% done | ETA(epoch): 1084s + [Epoch 247][20/66] loss=0.886827 avg=0.824903 VRAM=38.9GiB | 82.4% done | ETA(epoch): 890s + [Epoch 247][30/66] loss=0.973966 avg=0.833672 VRAM=39.0GiB | 82.5% done | ETA(epoch): 696s + [Epoch 247][40/66] loss=0.851722 avg=0.843668 VRAM=38.9GiB | 82.5% done | ETA(epoch): 503s + [Epoch 247][50/66] loss=0.804367 avg=0.844695 VRAM=39.0GiB | 82.6% done | ETA(epoch): 310s + [Epoch 247][60/66] loss=0.847013 avg=0.845613 VRAM=38.9GiB | 82.6% done | ETA(epoch): 116s + Train loss: 0.843244 (1279.4s) ETA: 1224min + Val loss: 0.841006 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0053 t_0.4-0.6=0.8541 t_0.6-0.8=0.7051 t_0.8-1.0=0.6310] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 248/299 (83% done) --- + [Epoch 248][10/66] loss=0.747102 avg=0.850394 VRAM=39.0GiB | 82.7% done | ETA(epoch): 1082s + [Epoch 248][20/66] loss=0.884974 avg=0.845383 VRAM=38.9GiB | 82.8% done | ETA(epoch): 889s + [Epoch 248][30/66] loss=0.928835 avg=0.846636 VRAM=39.0GiB | 82.8% done | ETA(epoch): 697s + [Epoch 248][40/66] loss=0.782555 avg=0.851300 VRAM=38.9GiB | 82.9% done | ETA(epoch): 503s + [Epoch 248][50/66] loss=0.842019 avg=0.846697 VRAM=39.0GiB | 82.9% done | ETA(epoch): 310s + [Epoch 248][60/66] loss=0.817228 avg=0.846769 VRAM=38.9GiB | 83.0% done | ETA(epoch): 116s + Train loss: 0.848675 (1280.9s) ETA: 1200min + Val loss: 0.866311 [t_0.0-0.2=1.0656 t_0.2-0.4=0.9603 t_0.4-0.6=0.8850 t_0.6-0.8=0.6906 t_0.8-1.0=0.6399] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0248 + Deleted old checkpoint: checkpoint_epoch_0245 +[MEM @ epoch 248 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 249/299 (83% done) --- + [Epoch 249][10/66] loss=0.795040 avg=0.846536 VRAM=39.0GiB | 83.1% done | ETA(epoch): 1083s + [Epoch 249][20/66] loss=0.810976 avg=0.859248 VRAM=38.9GiB | 83.1% done | ETA(epoch): 889s + [Epoch 249][30/66] loss=0.827083 avg=0.851199 VRAM=39.0GiB | 83.2% done | ETA(epoch): 696s + [Epoch 249][40/66] loss=0.780001 avg=0.850595 VRAM=38.9GiB | 83.2% done | ETA(epoch): 502s + [Epoch 249][50/66] loss=0.827490 avg=0.854322 VRAM=39.0GiB | 83.3% done | ETA(epoch): 310s + [Epoch 249][60/66] loss=1.010550 avg=0.855909 VRAM=38.9GiB | 83.3% done | ETA(epoch): 116s + Train loss: 0.853104 (1281.0s) ETA: 1177min + Val loss: 0.848558 [t_0.0-0.2=1.0893 t_0.2-0.4=0.9825 t_0.4-0.6=0.8552 t_0.6-0.8=0.7024 t_0.8-1.0=0.6229] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 21.8/188.5 GiB (11.5%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 250/299 (83% done) --- + [Epoch 250][10/66] loss=0.973270 avg=0.862590 VRAM=39.0GiB | 83.4% done | ETA(epoch): 1086s + [Epoch 250][20/66] loss=0.875572 avg=0.864526 VRAM=38.9GiB | 83.4% done | ETA(epoch): 898s + [Epoch 250][30/66] loss=0.942302 avg=0.854199 VRAM=39.0GiB | 83.5% done | ETA(epoch): 703s + [Epoch 250][40/66] loss=0.830379 avg=0.863052 VRAM=38.9GiB | 83.5% done | ETA(epoch): 506s + [Epoch 250][50/66] loss=0.730607 avg=0.865481 VRAM=39.0GiB | 83.6% done | ETA(epoch): 312s + [Epoch 250][60/66] loss=0.722024 avg=0.861146 VRAM=38.9GiB | 83.6% done | ETA(epoch): 117s + Train loss: 0.855866 (1282.5s) ETA: 1153min + Val loss: 0.873379 [t_0.0-0.2=1.0668 t_0.2-0.4=1.0096 t_0.4-0.6=0.8284 t_0.6-0.8=0.7090 t_0.8-1.0=0.6107] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 251/299 (84% done) --- + [Epoch 251][10/66] loss=0.805468 avg=0.838719 VRAM=39.0GiB | 83.7% done | ETA(epoch): 1085s + [Epoch 251][20/66] loss=0.913471 avg=0.879318 VRAM=38.9GiB | 83.8% done | ETA(epoch): 891s + [Epoch 251][30/66] loss=0.877857 avg=0.870434 VRAM=39.0GiB | 83.8% done | ETA(epoch): 699s + [Epoch 251][40/66] loss=0.809115 avg=0.858365 VRAM=38.9GiB | 83.9% done | ETA(epoch): 505s + [Epoch 251][50/66] loss=0.813393 avg=0.864121 VRAM=39.0GiB | 83.9% done | ETA(epoch): 310s + [Epoch 251][60/66] loss=0.710180 avg=0.854059 VRAM=38.9GiB | 84.0% done | ETA(epoch): 116s + Train loss: 0.852777 (1279.2s) ETA: 1130min + Val loss: 0.859203 [t_0.0-0.2=1.0700 t_0.2-0.4=1.0055 t_0.4-0.6=0.7871 t_0.6-0.8=0.7152 t_0.8-1.0=0.6261] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 252/299 (84% done) --- + [Epoch 252][10/66] loss=0.838012 avg=0.870636 VRAM=39.0GiB | 84.1% done | ETA(epoch): 1088s + [Epoch 252][20/66] loss=0.781375 avg=0.844343 VRAM=38.9GiB | 84.1% done | ETA(epoch): 893s + [Epoch 252][30/66] loss=0.889516 avg=0.845338 VRAM=39.0GiB | 84.2% done | ETA(epoch): 700s + [Epoch 252][40/66] loss=0.800656 avg=0.836667 VRAM=38.9GiB | 84.2% done | ETA(epoch): 506s + [Epoch 252][50/66] loss=0.794046 avg=0.846521 VRAM=39.0GiB | 84.3% done | ETA(epoch): 311s + [Epoch 252][60/66] loss=0.854189 avg=0.846955 VRAM=38.9GiB | 84.3% done | ETA(epoch): 117s + Train loss: 0.850401 (1282.0s) ETA: 1106min + Val loss: 0.849110 [t_0.0-0.2=1.0754 t_0.2-0.4=0.9870 t_0.4-0.6=0.8179 t_0.6-0.8=0.7066 t_0.8-1.0=0.6243] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 253/299 (84% done) --- + [Epoch 253][10/66] loss=0.752447 avg=0.863523 VRAM=39.0GiB | 84.4% done | ETA(epoch): 1082s + [Epoch 253][20/66] loss=0.964417 avg=0.853465 VRAM=38.9GiB | 84.4% done | ETA(epoch): 889s + [Epoch 253][30/66] loss=0.894271 avg=0.855901 VRAM=39.0GiB | 84.5% done | ETA(epoch): 696s + [Epoch 253][40/66] loss=0.739644 avg=0.849031 VRAM=38.9GiB | 84.5% done | ETA(epoch): 503s + [Epoch 253][50/66] loss=0.864364 avg=0.842418 VRAM=39.0GiB | 84.6% done | ETA(epoch): 310s + [Epoch 253][60/66] loss=0.925048 avg=0.848426 VRAM=38.9GiB | 84.6% done | ETA(epoch): 116s + Train loss: 0.846840 (1282.1s) ETA: 1082min + Val loss: 0.840466 [t_0.0-0.2=1.0650 t_0.2-0.4=0.9753 t_0.4-0.6=0.8385 t_0.6-0.8=0.6915 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 254/299 (85% done) --- + [Epoch 254][10/66] loss=0.915109 avg=0.884666 VRAM=39.0GiB | 84.7% done | ETA(epoch): 1084s + [Epoch 254][20/66] loss=0.856309 avg=0.879700 VRAM=38.9GiB | 84.8% done | ETA(epoch): 890s + [Epoch 254][30/66] loss=0.751409 avg=0.864209 VRAM=39.0GiB | 84.8% done | ETA(epoch): 697s + [Epoch 254][40/66] loss=0.807224 avg=0.862836 VRAM=38.9GiB | 84.9% done | ETA(epoch): 504s + [Epoch 254][50/66] loss=0.842932 avg=0.861812 VRAM=39.0GiB | 84.9% done | ETA(epoch): 310s + [Epoch 254][60/66] loss=0.896406 avg=0.858276 VRAM=38.9GiB | 85.0% done | ETA(epoch): 117s + Train loss: 0.855818 (1287.4s) ETA: 1059min + Val loss: 0.856006 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9841 t_0.4-0.6=0.8436 t_0.6-0.8=0.7039 t_0.8-1.0=0.6351] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 255/299 (85% done) --- + [Epoch 255][10/66] loss=0.847628 avg=0.867976 VRAM=39.0GiB | 85.1% done | ETA(epoch): 1084s + [Epoch 255][20/66] loss=0.900650 avg=0.859225 VRAM=38.9GiB | 85.1% done | ETA(epoch): 889s + [Epoch 255][30/66] loss=0.964900 avg=0.862985 VRAM=39.0GiB | 85.2% done | ETA(epoch): 696s + [Epoch 255][40/66] loss=1.041193 avg=0.857801 VRAM=38.9GiB | 85.2% done | ETA(epoch): 502s + [Epoch 255][50/66] loss=0.887748 avg=0.852683 VRAM=39.0GiB | 85.3% done | ETA(epoch): 309s + [Epoch 255][60/66] loss=0.949751 avg=0.853847 VRAM=38.9GiB | 85.3% done | ETA(epoch): 116s + Train loss: 0.852131 (1277.1s) ETA: 1035min + Val loss: 0.808385 [t_0.0-0.2=1.0618 t_0.2-0.4=0.9985 t_0.4-0.6=0.8363 t_0.6-0.8=0.7042 t_0.8-1.0=0.6216] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 256/299 (85% done) --- + [Epoch 256][10/66] loss=0.922014 avg=0.866587 VRAM=39.0GiB | 85.4% done | ETA(epoch): 1083s + [Epoch 256][20/66] loss=0.758063 avg=0.842806 VRAM=38.9GiB | 85.4% done | ETA(epoch): 891s + [Epoch 256][30/66] loss=0.869231 avg=0.846094 VRAM=39.0GiB | 85.5% done | ETA(epoch): 698s + [Epoch 256][40/66] loss=0.851291 avg=0.853765 VRAM=38.9GiB | 85.5% done | ETA(epoch): 506s + [Epoch 256][50/66] loss=0.719342 avg=0.852757 VRAM=39.0GiB | 85.6% done | ETA(epoch): 311s + [Epoch 256][60/66] loss=0.936071 avg=0.855252 VRAM=38.9GiB | 85.6% done | ETA(epoch): 116s + Train loss: 0.852657 (1279.8s) ETA: 1011min + Val loss: 0.832386 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9648 t_0.4-0.6=0.8625 t_0.6-0.8=0.7157 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 257/299 (86% done) --- + [Epoch 257][10/66] loss=0.833009 avg=0.854860 VRAM=39.0GiB | 85.7% done | ETA(epoch): 1081s + [Epoch 257][20/66] loss=0.940766 avg=0.852113 VRAM=38.9GiB | 85.8% done | ETA(epoch): 889s + [Epoch 257][30/66] loss=0.772359 avg=0.844308 VRAM=39.0GiB | 85.8% done | ETA(epoch): 696s + [Epoch 257][40/66] loss=0.899236 avg=0.856462 VRAM=38.9GiB | 85.9% done | ETA(epoch): 504s + [Epoch 257][50/66] loss=0.739082 avg=0.850101 VRAM=39.0GiB | 85.9% done | ETA(epoch): 310s + [Epoch 257][60/66] loss=0.707789 avg=0.848398 VRAM=38.9GiB | 86.0% done | ETA(epoch): 116s + Train loss: 0.851136 (1279.9s) ETA: 988min + Val loss: 0.833968 [t_0.0-0.2=1.0763 t_0.2-0.4=1.0213 t_0.4-0.6=0.8586 t_0.6-0.8=0.6871 t_0.8-1.0=0.6343] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 258/299 (86% done) --- + [Epoch 258][10/66] loss=0.940371 avg=0.864465 VRAM=39.0GiB | 86.1% done | ETA(epoch): 1086s + [Epoch 258][20/66] loss=0.856161 avg=0.861834 VRAM=38.9GiB | 86.1% done | ETA(epoch): 896s + [Epoch 258][30/66] loss=0.893610 avg=0.872076 VRAM=39.0GiB | 86.2% done | ETA(epoch): 699s + [Epoch 258][40/66] loss=0.873666 avg=0.879696 VRAM=38.9GiB | 86.2% done | ETA(epoch): 505s + [Epoch 258][50/66] loss=0.929420 avg=0.874971 VRAM=39.0GiB | 86.3% done | ETA(epoch): 310s + [Epoch 258][60/66] loss=0.841880 avg=0.866713 VRAM=38.9GiB | 86.3% done | ETA(epoch): 116s + Train loss: 0.861582 (1279.7s) ETA: 964min + Val loss: 0.862968 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9720 t_0.4-0.6=0.8428 t_0.6-0.8=0.7054 t_0.8-1.0=0.6272] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 259/299 (86% done) --- + [Epoch 259][10/66] loss=0.801863 avg=0.835841 VRAM=39.0GiB | 86.4% done | ETA(epoch): 1091s + [Epoch 259][20/66] loss=1.014709 avg=0.840780 VRAM=38.9GiB | 86.4% done | ETA(epoch): 893s + [Epoch 259][30/66] loss=0.960183 avg=0.836802 VRAM=39.0GiB | 86.5% done | ETA(epoch): 701s + [Epoch 259][40/66] loss=0.802611 avg=0.840452 VRAM=38.9GiB | 86.5% done | ETA(epoch): 506s + [Epoch 259][50/66] loss=0.744972 avg=0.840862 VRAM=39.0GiB | 86.6% done | ETA(epoch): 311s + [Epoch 259][60/66] loss=0.769481 avg=0.839041 VRAM=38.9GiB | 86.6% done | ETA(epoch): 117s + Train loss: 0.842410 (1281.4s) ETA: 941min + Val loss: 0.820570 [t_0.0-0.2=1.0713 t_0.2-0.4=1.0021 t_0.4-0.6=0.8545 t_0.6-0.8=0.7236 t_0.8-1.0=0.6360] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 260/299 (87% done) --- + [Epoch 260][10/66] loss=0.827809 avg=0.868214 VRAM=39.0GiB | 86.7% done | ETA(epoch): 1082s + [Epoch 260][20/66] loss=0.859626 avg=0.852690 VRAM=38.9GiB | 86.8% done | ETA(epoch): 889s + [Epoch 260][30/66] loss=0.902912 avg=0.854543 VRAM=39.0GiB | 86.8% done | ETA(epoch): 696s + [Epoch 260][40/66] loss=0.867152 avg=0.859559 VRAM=38.9GiB | 86.9% done | ETA(epoch): 503s + [Epoch 260][50/66] loss=0.909257 avg=0.863058 VRAM=39.0GiB | 86.9% done | ETA(epoch): 309s + [Epoch 260][60/66] loss=0.852421 avg=0.855983 VRAM=38.9GiB | 87.0% done | ETA(epoch): 116s + Train loss: 0.858590 (1276.7s) ETA: 917min + Val loss: 0.806741 [t_0.0-0.2=1.0947 t_0.2-0.4=0.9536 t_0.4-0.6=0.8376 t_0.6-0.8=0.6899 t_0.8-1.0=0.6427] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 261/299 (87% done) --- + [Epoch 261][10/66] loss=0.847214 avg=0.845608 VRAM=39.0GiB | 87.1% done | ETA(epoch): 1087s + [Epoch 261][20/66] loss=0.831323 avg=0.840624 VRAM=38.9GiB | 87.1% done | ETA(epoch): 892s + [Epoch 261][30/66] loss=0.813685 avg=0.838471 VRAM=39.0GiB | 87.2% done | ETA(epoch): 697s + [Epoch 261][40/66] loss=0.922660 avg=0.841845 VRAM=38.9GiB | 87.2% done | ETA(epoch): 503s + [Epoch 261][50/66] loss=0.909561 avg=0.844371 VRAM=39.0GiB | 87.3% done | ETA(epoch): 310s + [Epoch 261][60/66] loss=0.835021 avg=0.843979 VRAM=38.9GiB | 87.3% done | ETA(epoch): 116s + Train loss: 0.844457 (1278.3s) ETA: 893min + Val loss: 0.868601 [t_0.0-0.2=1.0681 t_0.2-0.4=1.0225 t_0.4-0.6=0.7939 t_0.6-0.8=0.7100 t_0.8-1.0=0.6200] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 262/299 (87% done) --- + [Epoch 262][10/66] loss=0.822990 avg=0.830919 VRAM=39.0GiB | 87.4% done | ETA(epoch): 1092s + [Epoch 262][20/66] loss=0.841496 avg=0.816744 VRAM=38.9GiB | 87.4% done | ETA(epoch): 894s + [Epoch 262][30/66] loss=0.795583 avg=0.834341 VRAM=39.0GiB | 87.5% done | ETA(epoch): 700s + [Epoch 262][40/66] loss=0.818388 avg=0.833497 VRAM=38.9GiB | 87.5% done | ETA(epoch): 505s + [Epoch 262][50/66] loss=0.920146 avg=0.848008 VRAM=39.0GiB | 87.6% done | ETA(epoch): 310s + [Epoch 262][60/66] loss=0.872748 avg=0.852641 VRAM=38.9GiB | 87.6% done | ETA(epoch): 116s + Train loss: 0.854658 (1280.5s) ETA: 870min + Val loss: 0.871494 [t_0.0-0.2=1.0664 t_0.2-0.4=0.9694 t_0.4-0.6=0.8278 t_0.6-0.8=0.7278 t_0.8-1.0=0.6180] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 263/299 (88% done) --- + [Epoch 263][10/66] loss=0.789866 avg=0.804861 VRAM=39.0GiB | 87.7% done | ETA(epoch): 1089s + [Epoch 263][20/66] loss=0.796514 avg=0.832313 VRAM=38.9GiB | 87.8% done | ETA(epoch): 893s + [Epoch 263][30/66] loss=0.790980 avg=0.839246 VRAM=39.0GiB | 87.8% done | ETA(epoch): 698s + [Epoch 263][40/66] loss=0.927862 avg=0.836376 VRAM=38.9GiB | 87.9% done | ETA(epoch): 506s + [Epoch 263][50/66] loss=0.938247 avg=0.848279 VRAM=39.0GiB | 87.9% done | ETA(epoch): 311s + [Epoch 263][60/66] loss=0.800288 avg=0.842646 VRAM=38.9GiB | 88.0% done | ETA(epoch): 117s + Train loss: 0.840537 (1281.4s) ETA: 846min + Val loss: 0.844382 [t_0.0-0.2=1.0585 t_0.2-0.4=1.0099 t_0.4-0.6=0.8483 t_0.6-0.8=0.7239 t_0.8-1.0=0.6186] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 264/299 (88% done) --- + [Epoch 264][10/66] loss=0.888244 avg=0.851306 VRAM=39.0GiB | 88.1% done | ETA(epoch): 1087s + [Epoch 264][20/66] loss=0.852708 avg=0.848580 VRAM=38.9GiB | 88.1% done | ETA(epoch): 895s + [Epoch 264][30/66] loss=0.834611 avg=0.853223 VRAM=39.0GiB | 88.2% done | ETA(epoch): 702s + [Epoch 264][40/66] loss=0.932683 avg=0.856927 VRAM=38.9GiB | 88.2% done | ETA(epoch): 506s + [Epoch 264][50/66] loss=0.817109 avg=0.852086 VRAM=39.0GiB | 88.3% done | ETA(epoch): 311s + [Epoch 264][60/66] loss=0.768720 avg=0.852280 VRAM=38.9GiB | 88.3% done | ETA(epoch): 117s + Train loss: 0.854475 (1284.1s) ETA: 823min + Val loss: 0.834966 [t_0.0-0.2=1.0746 t_0.2-0.4=1.0157 t_0.4-0.6=0.8509 t_0.6-0.8=0.6898 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 265/299 (88% done) --- + [Epoch 265][10/66] loss=0.949575 avg=0.902776 VRAM=39.0GiB | 88.4% done | ETA(epoch): 1090s + [Epoch 265][20/66] loss=0.873656 avg=0.880212 VRAM=38.9GiB | 88.4% done | ETA(epoch): 894s + [Epoch 265][30/66] loss=0.749650 avg=0.871893 VRAM=39.0GiB | 88.5% done | ETA(epoch): 699s + [Epoch 265][40/66] loss=0.915507 avg=0.861939 VRAM=38.9GiB | 88.5% done | ETA(epoch): 504s + [Epoch 265][50/66] loss=0.869043 avg=0.861437 VRAM=39.0GiB | 88.6% done | ETA(epoch): 310s + [Epoch 265][60/66] loss=0.831260 avg=0.856638 VRAM=38.9GiB | 88.6% done | ETA(epoch): 116s + Train loss: 0.852227 (1278.3s) ETA: 799min + Val loss: 0.818631 [t_0.0-0.2=1.0594 t_0.2-0.4=0.9953 t_0.4-0.6=0.8419 t_0.6-0.8=0.7007 t_0.8-1.0=0.6116] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 266/299 (89% done) --- + [Epoch 266][10/66] loss=0.855767 avg=0.849427 VRAM=39.0GiB | 88.7% done | ETA(epoch): 1087s + [Epoch 266][20/66] loss=0.967624 avg=0.854185 VRAM=38.9GiB | 88.8% done | ETA(epoch): 892s + [Epoch 266][30/66] loss=0.756747 avg=0.847812 VRAM=39.0GiB | 88.8% done | ETA(epoch): 698s + [Epoch 266][40/66] loss=0.773560 avg=0.852361 VRAM=38.9GiB | 88.9% done | ETA(epoch): 504s + [Epoch 266][50/66] loss=0.775835 avg=0.853551 VRAM=39.0GiB | 88.9% done | ETA(epoch): 310s + [Epoch 266][60/66] loss=0.688472 avg=0.853182 VRAM=38.9GiB | 89.0% done | ETA(epoch): 116s + Train loss: 0.847674 (1280.9s) ETA: 775min + Val loss: 0.856156 [t_0.0-0.2=1.0867 t_0.2-0.4=0.9933 t_0.4-0.6=0.8329 t_0.6-0.8=0.7079 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 267/299 (89% done) --- + [Epoch 267][10/66] loss=0.927769 avg=0.832598 VRAM=39.0GiB | 89.1% done | ETA(epoch): 1088s + [Epoch 267][20/66] loss=0.767195 avg=0.830248 VRAM=38.9GiB | 89.1% done | ETA(epoch): 893s + [Epoch 267][30/66] loss=0.798127 avg=0.845566 VRAM=39.0GiB | 89.2% done | ETA(epoch): 698s + [Epoch 267][40/66] loss=0.821625 avg=0.840925 VRAM=38.9GiB | 89.2% done | ETA(epoch): 504s + [Epoch 267][50/66] loss=0.760634 avg=0.838878 VRAM=39.0GiB | 89.3% done | ETA(epoch): 311s + [Epoch 267][60/66] loss=0.803537 avg=0.839954 VRAM=38.9GiB | 89.3% done | ETA(epoch): 116s + Train loss: 0.846584 (1288.2s) ETA: 752min + Val loss: 0.854375 [t_0.0-0.2=1.0529 t_0.2-0.4=1.0087 t_0.4-0.6=0.8428 t_0.6-0.8=0.6970 t_0.8-1.0=0.6322] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 268/299 (89% done) --- + [Epoch 268][10/66] loss=0.870314 avg=0.878165 VRAM=39.0GiB | 89.4% done | ETA(epoch): 1088s + [Epoch 268][20/66] loss=0.962785 avg=0.858829 VRAM=38.9GiB | 89.4% done | ETA(epoch): 895s + [Epoch 268][30/66] loss=0.846462 avg=0.855220 VRAM=39.0GiB | 89.5% done | ETA(epoch): 700s + [Epoch 268][40/66] loss=0.834944 avg=0.854542 VRAM=38.9GiB | 89.5% done | ETA(epoch): 506s + [Epoch 268][50/66] loss=0.925180 avg=0.859072 VRAM=39.0GiB | 89.6% done | ETA(epoch): 311s + [Epoch 268][60/66] loss=0.734652 avg=0.850600 VRAM=38.9GiB | 89.6% done | ETA(epoch): 117s + Train loss: 0.851617 (1285.7s) ETA: 728min + Val loss: 0.848961 [t_0.0-0.2=1.0649 t_0.2-0.4=1.0219 t_0.4-0.6=0.8770 t_0.6-0.8=0.6834 t_0.8-1.0=0.6359] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 269/299 (90% done) --- + [Epoch 269][10/66] loss=0.889495 avg=0.867279 VRAM=39.0GiB | 89.7% done | ETA(epoch): 1084s + [Epoch 269][20/66] loss=0.934194 avg=0.862536 VRAM=38.9GiB | 89.8% done | ETA(epoch): 890s + [Epoch 269][30/66] loss=0.808582 avg=0.863326 VRAM=39.0GiB | 89.8% done | ETA(epoch): 696s + [Epoch 269][40/66] loss=0.839940 avg=0.854528 VRAM=38.9GiB | 89.9% done | ETA(epoch): 503s + [Epoch 269][50/66] loss=0.750920 avg=0.850483 VRAM=39.0GiB | 89.9% done | ETA(epoch): 310s + [Epoch 269][60/66] loss=0.924932 avg=0.845850 VRAM=38.9GiB | 90.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 17819 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 17819 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 17819 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 17819 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 17819 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 17819 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 17819 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 17819 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 17819 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 17819 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.842802 (2332.9s) ETA: 707min + Val loss: 0.856293 [t_0.0-0.2=1.0929 t_0.2-0.4=1.0137 t_0.4-0.6=0.8645 t_0.6-0.8=0.6822 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 22.6/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 270/299 (90% done) --- + [Epoch 270][10/66] loss=0.922426 avg=0.891219 VRAM=39.0GiB | 90.1% done | ETA(epoch): 1093s + [Epoch 270][20/66] loss=0.888938 avg=0.864856 VRAM=38.9GiB | 90.1% done | ETA(epoch): 895s + [Epoch 270][30/66] loss=0.858802 avg=0.866808 VRAM=39.0GiB | 90.2% done | ETA(epoch): 699s + [Epoch 270][40/66] loss=0.880181 avg=0.857541 VRAM=38.9GiB | 90.2% done | ETA(epoch): 505s + [Epoch 270][50/66] loss=0.729378 avg=0.843937 VRAM=39.0GiB | 90.3% done | ETA(epoch): 312s + [Epoch 270][60/66] loss=0.755898 avg=0.836696 VRAM=38.9GiB | 90.3% done | ETA(epoch): 117s + Train loss: 0.835124 (1284.8s) ETA: 683min + Val loss: 0.830426 [t_0.0-0.2=1.0839 t_0.2-0.4=0.9582 t_0.4-0.6=0.8429 t_0.6-0.8=0.7381 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 271/299 (90% done) --- + [Epoch 271][10/66] loss=0.950175 avg=0.879949 VRAM=39.0GiB | 90.4% done | ETA(epoch): 1086s + [Epoch 271][20/66] loss=0.833677 avg=0.836404 VRAM=38.9GiB | 90.4% done | ETA(epoch): 892s + [Epoch 271][30/66] loss=0.880122 avg=0.835670 VRAM=39.0GiB | 90.5% done | ETA(epoch): 699s + [Epoch 271][40/66] loss=0.747319 avg=0.843615 VRAM=38.9GiB | 90.5% done | ETA(epoch): 504s + [Epoch 271][50/66] loss=0.841239 avg=0.845461 VRAM=39.0GiB | 90.6% done | ETA(epoch): 311s + [Epoch 271][60/66] loss=0.779256 avg=0.841400 VRAM=38.9GiB | 90.6% done | ETA(epoch): 117s + Train loss: 0.839937 (1281.7s) ETA: 660min + Val loss: 0.847575 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9952 t_0.4-0.6=0.8453 t_0.6-0.8=0.6837 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 272/299 (91% done) --- + [Epoch 272][10/66] loss=0.844201 avg=0.814135 VRAM=39.0GiB | 90.7% done | ETA(epoch): 1083s + [Epoch 272][20/66] loss=0.926095 avg=0.840151 VRAM=38.9GiB | 90.8% done | ETA(epoch): 890s + [Epoch 272][30/66] loss=0.885499 avg=0.847690 VRAM=39.0GiB | 90.8% done | ETA(epoch): 699s + [Epoch 272][40/66] loss=0.971784 avg=0.846977 VRAM=38.9GiB | 90.9% done | ETA(epoch): 505s + [Epoch 272][50/66] loss=0.928664 avg=0.847043 VRAM=39.0GiB | 90.9% done | ETA(epoch): 311s + [Epoch 272][60/66] loss=0.850168 avg=0.851512 VRAM=38.9GiB | 91.0% done | ETA(epoch): 117s + Train loss: 0.852436 (1283.4s) ETA: 636min + Val loss: 0.821639 [t_0.0-0.2=1.0681 t_0.2-0.4=0.9605 t_0.4-0.6=0.8310 t_0.6-0.8=0.7177 t_0.8-1.0=0.6171] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 273/299 (91% done) --- + [Epoch 273][10/66] loss=0.759353 avg=0.870383 VRAM=39.0GiB | 91.1% done | ETA(epoch): 1084s + [Epoch 273][20/66] loss=0.810179 avg=0.855348 VRAM=38.9GiB | 91.1% done | ETA(epoch): 891s + [Epoch 273][30/66] loss=0.952095 avg=0.851969 VRAM=39.0GiB | 91.2% done | ETA(epoch): 697s + [Epoch 273][40/66] loss=0.900437 avg=0.845710 VRAM=38.9GiB | 91.2% done | ETA(epoch): 503s + [Epoch 273][50/66] loss=0.951690 avg=0.847992 VRAM=39.0GiB | 91.3% done | ETA(epoch): 310s + [Epoch 273][60/66] loss=0.825505 avg=0.845809 VRAM=38.9GiB | 91.3% done | ETA(epoch): 116s + Train loss: 0.844052 (1277.9s) ETA: 612min + Val loss: 0.821787 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9751 t_0.4-0.6=0.8385 t_0.6-0.8=0.7006 t_0.8-1.0=0.6393] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 274/299 (91% done) --- + [Epoch 274][10/66] loss=0.878517 avg=0.829559 VRAM=39.0GiB | 91.4% done | ETA(epoch): 1102s + [Epoch 274][20/66] loss=0.699867 avg=0.833064 VRAM=38.9GiB | 91.4% done | ETA(epoch): 898s + [Epoch 274][30/66] loss=0.940496 avg=0.830651 VRAM=39.0GiB | 91.5% done | ETA(epoch): 702s + [Epoch 274][40/66] loss=0.717877 avg=0.836357 VRAM=38.9GiB | 91.5% done | ETA(epoch): 506s + [Epoch 274][50/66] loss=0.844635 avg=0.842702 VRAM=39.0GiB | 91.6% done | ETA(epoch): 311s + [Epoch 274][60/66] loss=0.791161 avg=0.842718 VRAM=38.9GiB | 91.6% done | ETA(epoch): 117s + Train loss: 0.844911 (1282.3s) ETA: 589min + Val loss: 0.861398 [t_0.0-0.2=1.0906 t_0.2-0.4=0.9686 t_0.4-0.6=0.8265 t_0.6-0.8=0.7224 t_0.8-1.0=0.6317] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 22.6/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 275/299 (92% done) --- + [Epoch 275][10/66] loss=0.790925 avg=0.841714 VRAM=39.0GiB | 91.7% done | ETA(epoch): 1092s + [Epoch 275][20/66] loss=0.812257 avg=0.831465 VRAM=38.9GiB | 91.8% done | ETA(epoch): 897s + [Epoch 275][30/66] loss=0.801225 avg=0.829584 VRAM=39.0GiB | 91.8% done | ETA(epoch): 702s + [Epoch 275][40/66] loss=0.944365 avg=0.836732 VRAM=38.9GiB | 91.9% done | ETA(epoch): 506s + [Epoch 275][50/66] loss=0.840185 avg=0.839956 VRAM=39.0GiB | 91.9% done | ETA(epoch): 311s + [Epoch 275][60/66] loss=0.711020 avg=0.842082 VRAM=38.9GiB | 92.0% done | ETA(epoch): 117s + Train loss: 0.844976 (1281.7s) ETA: 565min + Val loss: 0.859216 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9952 t_0.4-0.6=0.8786 t_0.6-0.8=0.6815 t_0.8-1.0=0.6249] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 22.5/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 276/299 (92% done) --- + [Epoch 276][10/66] loss=0.674339 avg=0.781560 VRAM=39.0GiB | 92.1% done | ETA(epoch): 1084s + [Epoch 276][20/66] loss=0.839017 avg=0.828781 VRAM=38.9GiB | 92.1% done | ETA(epoch): 890s + [Epoch 276][30/66] loss=0.800607 avg=0.826933 VRAM=39.0GiB | 92.2% done | ETA(epoch): 697s + [Epoch 276][40/66] loss=0.905132 avg=0.832122 VRAM=38.9GiB | 92.2% done | ETA(epoch): 504s + [Epoch 276][50/66] loss=0.835124 avg=0.832463 VRAM=39.0GiB | 92.3% done | ETA(epoch): 311s + [Epoch 276][60/66] loss=0.762985 avg=0.837582 VRAM=38.9GiB | 92.3% done | ETA(epoch): 117s + Train loss: 0.838569 (1281.7s) ETA: 541min + Val loss: 0.836416 [t_0.0-0.2=1.0665 t_0.2-0.4=1.0079 t_0.4-0.6=0.8346 t_0.6-0.8=0.6839 t_0.8-1.0=0.6279] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 277/299 (92% done) --- + [Epoch 277][10/66] loss=0.810480 avg=0.859918 VRAM=39.0GiB | 92.4% done | ETA(epoch): 1084s + [Epoch 277][20/66] loss=0.805557 avg=0.847179 VRAM=38.9GiB | 92.4% done | ETA(epoch): 890s + [Epoch 277][30/66] loss=0.719766 avg=0.839302 VRAM=39.0GiB | 92.5% done | ETA(epoch): 697s + [Epoch 277][40/66] loss=0.899786 avg=0.837713 VRAM=38.9GiB | 92.5% done | ETA(epoch): 503s + [Epoch 277][50/66] loss=0.787502 avg=0.837537 VRAM=39.0GiB | 92.6% done | ETA(epoch): 310s + [Epoch 277][60/66] loss=0.893699 avg=0.837682 VRAM=38.9GiB | 92.6% done | ETA(epoch): 116s + Train loss: 0.840704 (1278.4s) ETA: 518min + Val loss: 0.874822 [t_0.0-0.2=1.0700 t_0.2-0.4=1.0184 t_0.4-0.6=0.8631 t_0.6-0.8=0.6991 t_0.8-1.0=0.6427] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 278/299 (93% done) --- + [Epoch 278][10/66] loss=0.769766 avg=0.867400 VRAM=39.0GiB | 92.7% done | ETA(epoch): 1091s + [Epoch 278][20/66] loss=0.835104 avg=0.860669 VRAM=38.9GiB | 92.8% done | ETA(epoch): 897s + [Epoch 278][30/66] loss=0.825231 avg=0.844232 VRAM=39.0GiB | 92.8% done | ETA(epoch): 701s + [Epoch 278][40/66] loss=0.818682 avg=0.852827 VRAM=38.9GiB | 92.9% done | ETA(epoch): 506s + [Epoch 278][50/66] loss=0.987850 avg=0.852421 VRAM=39.0GiB | 92.9% done | ETA(epoch): 311s + [Epoch 278][60/66] loss=0.844683 avg=0.850902 VRAM=38.9GiB | 93.0% done | ETA(epoch): 117s + Train loss: 0.851569 (1281.9s) ETA: 494min + Val loss: 0.873805 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9893 t_0.4-0.6=0.8465 t_0.6-0.8=0.6857 t_0.8-1.0=0.6076] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 22.5/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 279/299 (93% done) --- + [Epoch 279][10/66] loss=0.975373 avg=0.847055 VRAM=39.0GiB | 93.1% done | ETA(epoch): 1083s + [Epoch 279][20/66] loss=0.834555 avg=0.860568 VRAM=38.9GiB | 93.1% done | ETA(epoch): 891s + [Epoch 279][30/66] loss=0.904895 avg=0.862712 VRAM=39.0GiB | 93.2% done | ETA(epoch): 699s + [Epoch 279][40/66] loss=1.025148 avg=0.871461 VRAM=38.9GiB | 93.2% done | ETA(epoch): 504s + [Epoch 279][50/66] loss=0.724960 avg=0.864472 VRAM=39.0GiB | 93.3% done | ETA(epoch): 310s + [Epoch 279][60/66] loss=0.785547 avg=0.860395 VRAM=38.9GiB | 93.3% done | ETA(epoch): 116s + Train loss: 0.862231 (1281.8s) ETA: 471min + Val loss: 0.855611 [t_0.0-0.2=1.0684 t_0.2-0.4=0.9996 t_0.4-0.6=0.8324 t_0.6-0.8=0.7224 t_0.8-1.0=0.6298] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 280/299 (93% done) --- + [Epoch 280][10/66] loss=0.955392 avg=0.854334 VRAM=39.0GiB | 93.4% done | ETA(epoch): 1084s + [Epoch 280][20/66] loss=0.749626 avg=0.864306 VRAM=38.9GiB | 93.4% done | ETA(epoch): 904s + [Epoch 280][30/66] loss=0.822963 avg=0.857637 VRAM=39.0GiB | 93.5% done | ETA(epoch): 704s + [Epoch 280][40/66] loss=0.730812 avg=0.848296 VRAM=38.9GiB | 93.5% done | ETA(epoch): 507s + [Epoch 280][50/66] loss=0.857910 avg=0.849346 VRAM=39.0GiB | 93.6% done | ETA(epoch): 312s + [Epoch 280][60/66] loss=0.980958 avg=0.850750 VRAM=38.9GiB | 93.6% done | ETA(epoch): 117s + Train loss: 0.848936 (1284.2s) ETA: 447min + Val loss: 0.874429 [t_0.0-0.2=1.0803 t_0.2-0.4=0.9686 t_0.4-0.6=0.8548 t_0.6-0.8=0.7177 t_0.8-1.0=0.6289] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 281/299 (94% done) --- + [Epoch 281][10/66] loss=0.834865 avg=0.820122 VRAM=39.0GiB | 93.7% done | ETA(epoch): 1094s + [Epoch 281][20/66] loss=0.784139 avg=0.841684 VRAM=38.9GiB | 93.8% done | ETA(epoch): 895s + [Epoch 281][30/66] loss=0.733421 avg=0.847032 VRAM=39.0GiB | 93.8% done | ETA(epoch): 700s + [Epoch 281][40/66] loss=0.780475 avg=0.848455 VRAM=38.9GiB | 93.9% done | ETA(epoch): 505s + [Epoch 281][50/66] loss=0.872510 avg=0.853834 VRAM=39.0GiB | 93.9% done | ETA(epoch): 311s + [Epoch 281][60/66] loss=0.861208 avg=0.854073 VRAM=38.9GiB | 94.0% done | ETA(epoch): 117s + Train loss: 0.857217 (1282.9s) ETA: 424min + Val loss: 0.829968 [t_0.0-0.2=1.0743 t_0.2-0.4=0.9720 t_0.4-0.6=0.8315 t_0.6-0.8=0.7055 t_0.8-1.0=0.6246] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0281 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 282/299 (94% done) --- + [Epoch 282][10/66] loss=0.861096 avg=0.840211 VRAM=39.0GiB | 94.1% done | ETA(epoch): 1084s + [Epoch 282][20/66] loss=0.823945 avg=0.857190 VRAM=38.9GiB | 94.1% done | ETA(epoch): 891s + [Epoch 282][30/66] loss=0.962236 avg=0.856434 VRAM=39.0GiB | 94.2% done | ETA(epoch): 697s + [Epoch 282][40/66] loss=0.858311 avg=0.858661 VRAM=38.9GiB | 94.2% done | ETA(epoch): 504s + [Epoch 282][50/66] loss=0.803073 avg=0.859439 VRAM=39.0GiB | 94.3% done | ETA(epoch): 310s + [Epoch 282][60/66] loss=0.902975 avg=0.856264 VRAM=38.9GiB | 94.3% done | ETA(epoch): 116s + Train loss: 0.856959 (1280.5s) ETA: 400min + Val loss: 0.843636 [t_0.0-0.2=1.0577 t_0.2-0.4=0.9948 t_0.4-0.6=0.8656 t_0.6-0.8=0.7003 t_0.8-1.0=0.6342] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 283/299 (94% done) --- + [Epoch 283][10/66] loss=0.805282 avg=0.895414 VRAM=39.0GiB | 94.4% done | ETA(epoch): 1089s + [Epoch 283][20/66] loss=0.877868 avg=0.880091 VRAM=38.9GiB | 94.4% done | ETA(epoch): 892s + [Epoch 283][30/66] loss=0.796373 avg=0.866415 VRAM=39.0GiB | 94.5% done | ETA(epoch): 698s + [Epoch 283][40/66] loss=0.831155 avg=0.863492 VRAM=38.9GiB | 94.5% done | ETA(epoch): 504s + [Epoch 283][50/66] loss=0.865659 avg=0.850182 VRAM=39.0GiB | 94.6% done | ETA(epoch): 310s + [Epoch 283][60/66] loss=0.989740 avg=0.853770 VRAM=38.9GiB | 94.6% done | ETA(epoch): 116s + Train loss: 0.851578 (1281.3s) ETA: 376min + Val loss: 0.857790 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9749 t_0.4-0.6=0.8567 t_0.6-0.8=0.6989 t_0.8-1.0=0.6271] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 22.6/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 284/299 (95% done) --- + [Epoch 284][10/66] loss=0.811107 avg=0.803392 VRAM=39.0GiB | 94.7% done | ETA(epoch): 1087s + [Epoch 284][20/66] loss=0.849988 avg=0.822765 VRAM=38.9GiB | 94.8% done | ETA(epoch): 892s + [Epoch 284][30/66] loss=0.959860 avg=0.844296 VRAM=39.0GiB | 94.8% done | ETA(epoch): 699s + [Epoch 284][40/66] loss=0.820751 avg=0.847188 VRAM=38.9GiB | 94.9% done | ETA(epoch): 505s + [Epoch 284][50/66] loss=0.906866 avg=0.844701 VRAM=39.0GiB | 94.9% done | ETA(epoch): 311s + [Epoch 284][60/66] loss=0.937800 avg=0.850331 VRAM=38.9GiB | 95.0% done | ETA(epoch): 117s + Train loss: 0.847404 (1282.8s) ETA: 353min + Val loss: 0.850153 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9945 t_0.4-0.6=0.8534 t_0.6-0.8=0.7280 t_0.8-1.0=0.6298] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0284 + Deleted old checkpoint: checkpoint_epoch_0281 +[MEM @ epoch 284 end] RAM: 22.6/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 285/299 (95% done) --- + [Epoch 285][10/66] loss=0.816319 avg=0.800904 VRAM=39.0GiB | 95.1% done | ETA(epoch): 1087s + [Epoch 285][20/66] loss=0.903947 avg=0.821063 VRAM=38.9GiB | 95.1% done | ETA(epoch): 892s + [Epoch 285][30/66] loss=0.917323 avg=0.823819 VRAM=39.0GiB | 95.2% done | ETA(epoch): 698s + [Epoch 285][40/66] loss=0.819888 avg=0.821104 VRAM=38.9GiB | 95.2% done | ETA(epoch): 504s + [Epoch 285][50/66] loss=0.789054 avg=0.825378 VRAM=39.0GiB | 95.3% done | ETA(epoch): 310s + [Epoch 285][60/66] loss=0.858422 avg=0.831811 VRAM=38.9GiB | 95.3% done | ETA(epoch): 116s + Train loss: 0.837450 (1278.2s) ETA: 329min + Val loss: 0.787668 [t_0.0-0.2=1.0493 t_0.2-0.4=1.0018 t_0.4-0.6=0.8068 t_0.6-0.8=0.7211 t_0.8-1.0=0.6279] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0285 (BEST) + Deleted old checkpoint: checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 22.7/188.5 GiB (12.1%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 286/299 (95% done) --- + [Epoch 286][10/66] loss=0.798163 avg=0.840064 VRAM=39.0GiB | 95.4% done | ETA(epoch): 1103s + [Epoch 286][20/66] loss=0.698622 avg=0.811720 VRAM=38.9GiB | 95.4% done | ETA(epoch): 899s + [Epoch 286][30/66] loss=0.925013 avg=0.829998 VRAM=39.0GiB | 95.5% done | ETA(epoch): 707s + [Epoch 286][40/66] loss=0.871085 avg=0.822450 VRAM=38.9GiB | 95.5% done | ETA(epoch): 510s + [Epoch 286][50/66] loss=0.849682 avg=0.825983 VRAM=39.0GiB | 95.6% done | ETA(epoch): 313s + [Epoch 286][60/66] loss=0.888282 avg=0.826964 VRAM=38.9GiB | 95.6% done | ETA(epoch): 117s + Train loss: 0.831106 (1287.6s) ETA: 306min + Val loss: 0.839310 [t_0.0-0.2=1.0830 t_0.2-0.4=1.0061 t_0.4-0.6=0.8681 t_0.6-0.8=0.6923 t_0.8-1.0=0.6243] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 22.6/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 287/299 (96% done) --- + [Epoch 287][10/66] loss=0.756994 avg=0.818512 VRAM=39.0GiB | 95.7% done | ETA(epoch): 1087s + [Epoch 287][20/66] loss=0.798908 avg=0.813608 VRAM=38.9GiB | 95.8% done | ETA(epoch): 897s + [Epoch 287][30/66] loss=0.869501 avg=0.837667 VRAM=39.0GiB | 95.8% done | ETA(epoch): 700s + [Epoch 287][40/66] loss=0.790982 avg=0.845335 VRAM=38.9GiB | 95.9% done | ETA(epoch): 505s + [Epoch 287][50/66] loss=0.962473 avg=0.843685 VRAM=39.0GiB | 95.9% done | ETA(epoch): 311s + [Epoch 287][60/66] loss=0.802145 avg=0.840371 VRAM=38.9GiB | 96.0% done | ETA(epoch): 116s + Train loss: 0.836988 (1280.5s) ETA: 282min + Val loss: 0.855945 [t_0.0-0.2=1.0668 t_0.2-0.4=0.9844 t_0.4-0.6=0.8687 t_0.6-0.8=0.7125 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 22.6/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 288/299 (96% done) --- + [Epoch 288][10/66] loss=0.822621 avg=0.831770 VRAM=39.0GiB | 96.1% done | ETA(epoch): 1130s + [Epoch 288][20/66] loss=0.925651 avg=0.851567 VRAM=38.9GiB | 96.1% done | ETA(epoch): 911s + [Epoch 288][30/66] loss=0.917099 avg=0.847017 VRAM=39.0GiB | 96.2% done | ETA(epoch): 709s + [Epoch 288][40/66] loss=0.803894 avg=0.844440 VRAM=38.9GiB | 96.2% done | ETA(epoch): 511s + [Epoch 288][50/66] loss=0.888780 avg=0.843146 VRAM=39.0GiB | 96.3% done | ETA(epoch): 314s + [Epoch 288][60/66] loss=0.803735 avg=0.843550 VRAM=38.9GiB | 96.3% done | ETA(epoch): 118s + Train loss: 0.843098 (1291.7s) ETA: 259min + Val loss: 0.824324 [t_0.0-0.2=1.0762 t_0.2-0.4=0.9608 t_0.4-0.6=0.8300 t_0.6-0.8=0.6956 t_0.8-1.0=0.6203] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0288 +[MEM @ epoch 288 end] RAM: 22.7/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 289/299 (96% done) --- + [Epoch 289][10/66] loss=0.986104 avg=0.846879 VRAM=39.0GiB | 96.4% done | ETA(epoch): 1092s + [Epoch 289][20/66] loss=0.902687 avg=0.847104 VRAM=38.9GiB | 96.4% done | ETA(epoch): 896s + [Epoch 289][30/66] loss=0.835636 avg=0.852015 VRAM=39.0GiB | 96.5% done | ETA(epoch): 702s + [Epoch 289][40/66] loss=0.938756 avg=0.855246 VRAM=38.9GiB | 96.5% done | ETA(epoch): 506s + [Epoch 289][50/66] loss=0.789343 avg=0.850506 VRAM=39.0GiB | 96.6% done | ETA(epoch): 311s + [Epoch 289][60/66] loss=0.782364 avg=0.848686 VRAM=38.9GiB | 96.6% done | ETA(epoch): 117s + Train loss: 0.847154 (1288.3s) ETA: 235min + Val loss: 0.885160 [t_0.0-0.2=1.0691 t_0.2-0.4=1.0274 t_0.4-0.6=0.7864 t_0.6-0.8=0.7152 t_0.8-1.0=0.6225] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 290/299 (97% done) --- + [Epoch 290][10/66] loss=0.759206 avg=0.875079 VRAM=39.0GiB | 96.7% done | ETA(epoch): 1086s + [Epoch 290][20/66] loss=0.904598 avg=0.871018 VRAM=38.9GiB | 96.8% done | ETA(epoch): 892s + [Epoch 290][30/66] loss=0.790905 avg=0.857479 VRAM=39.0GiB | 96.8% done | ETA(epoch): 698s + [Epoch 290][40/66] loss=0.886050 avg=0.850264 VRAM=38.9GiB | 96.9% done | ETA(epoch): 504s + [Epoch 290][50/66] loss=0.872599 avg=0.842692 VRAM=39.0GiB | 96.9% done | ETA(epoch): 311s + [Epoch 290][60/66] loss=0.893821 avg=0.849486 VRAM=38.9GiB | 97.0% done | ETA(epoch): 117s + Train loss: 0.846852 (1286.2s) ETA: 212min + Val loss: 0.837763 [t_0.0-0.2=1.0947 t_0.2-0.4=0.9804 t_0.4-0.6=0.8472 t_0.6-0.8=0.6917 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 22.6/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 291/299 (97% done) --- + [Epoch 291][10/66] loss=0.838975 avg=0.892030 VRAM=39.0GiB | 97.1% done | ETA(epoch): 1086s + [Epoch 291][20/66] loss=0.925175 avg=0.868890 VRAM=38.9GiB | 97.1% done | ETA(epoch): 894s + [Epoch 291][30/66] loss=0.750609 avg=0.851647 VRAM=39.0GiB | 97.2% done | ETA(epoch): 703s + [Epoch 291][40/66] loss=0.925535 avg=0.855503 VRAM=38.9GiB | 97.2% done | ETA(epoch): 507s + [Epoch 291][50/66] loss=0.769021 avg=0.849651 VRAM=39.0GiB | 97.3% done | ETA(epoch): 312s + [Epoch 291][60/66] loss=0.785513 avg=0.846420 VRAM=38.9GiB | 97.3% done | ETA(epoch): 117s + Train loss: 0.843418 (1285.0s) ETA: 188min + Val loss: 0.842834 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9990 t_0.4-0.6=0.8552 t_0.6-0.8=0.7032 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 22.5/188.5 GiB (12.0%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 292/299 (97% done) --- + [Epoch 292][10/66] loss=0.797007 avg=0.843848 VRAM=39.0GiB | 97.4% done | ETA(epoch): 1096s + [Epoch 292][20/66] loss=0.910505 avg=0.866681 VRAM=38.9GiB | 97.4% done | ETA(epoch): 896s + [Epoch 292][30/66] loss=0.913550 avg=0.857997 VRAM=39.0GiB | 97.5% done | ETA(epoch): 700s + [Epoch 292][40/66] loss=0.871908 avg=0.859666 VRAM=38.9GiB | 97.5% done | ETA(epoch): 507s + [Epoch 292][50/66] loss=0.834064 avg=0.865458 VRAM=39.0GiB | 97.6% done | ETA(epoch): 312s + [Epoch 292][60/66] loss=0.787763 avg=0.867220 VRAM=38.9GiB | 97.6% done | ETA(epoch): 117s + Train loss: 0.864700 (1284.5s) ETA: 165min + Val loss: 0.832126 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9918 t_0.4-0.6=0.8730 t_0.6-0.8=0.7105 t_0.8-1.0=0.6238] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 293/299 (98% done) --- + [Epoch 293][10/66] loss=0.795162 avg=0.843062 VRAM=39.0GiB | 97.7% done | ETA(epoch): 1093s + [Epoch 293][20/66] loss=0.725906 avg=0.820603 VRAM=38.9GiB | 97.8% done | ETA(epoch): 896s + [Epoch 293][30/66] loss=0.823680 avg=0.832564 VRAM=39.0GiB | 97.8% done | ETA(epoch): 704s + [Epoch 293][40/66] loss=0.950911 avg=0.837718 VRAM=38.9GiB | 97.9% done | ETA(epoch): 508s + [Epoch 293][50/66] loss=0.812615 avg=0.838367 VRAM=39.0GiB | 97.9% done | ETA(epoch): 312s + [Epoch 293][60/66] loss=0.890404 avg=0.840039 VRAM=38.9GiB | 98.0% done | ETA(epoch): 117s + Train loss: 0.839629 (1285.7s) ETA: 141min + Val loss: 0.867556 [t_0.0-0.2=1.0705 t_0.2-0.4=0.9933 t_0.4-0.6=0.8329 t_0.6-0.8=0.7153 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 22.3/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 294/299 (98% done) --- + [Epoch 294][10/66] loss=0.929858 avg=0.813121 VRAM=39.0GiB | 98.1% done | ETA(epoch): 1086s + [Epoch 294][20/66] loss=0.828935 avg=0.814062 VRAM=38.9GiB | 98.1% done | ETA(epoch): 896s + [Epoch 294][30/66] loss=0.741170 avg=0.830276 VRAM=39.0GiB | 98.2% done | ETA(epoch): 701s + [Epoch 294][40/66] loss=0.789940 avg=0.834230 VRAM=38.9GiB | 98.2% done | ETA(epoch): 506s + [Epoch 294][50/66] loss=0.910324 avg=0.838038 VRAM=39.0GiB | 98.3% done | ETA(epoch): 311s + [Epoch 294][60/66] loss=0.749614 avg=0.847057 VRAM=38.9GiB | 98.3% done | ETA(epoch): 117s + Train loss: 0.847838 (1283.4s) ETA: 118min + Val loss: 0.860483 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9738 t_0.4-0.6=0.8463 t_0.6-0.8=0.6989 t_0.8-1.0=0.6569] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 295/299 (98% done) --- + [Epoch 295][10/66] loss=0.957095 avg=0.839599 VRAM=39.0GiB | 98.4% done | ETA(epoch): 1087s + [Epoch 295][20/66] loss=0.899622 avg=0.842834 VRAM=38.9GiB | 98.4% done | ETA(epoch): 897s + [Epoch 295][30/66] loss=0.815742 avg=0.846257 VRAM=39.0GiB | 98.5% done | ETA(epoch): 701s + [Epoch 295][40/66] loss=0.950607 avg=0.854715 VRAM=38.9GiB | 98.5% done | ETA(epoch): 506s + [Epoch 295][50/66] loss=0.845682 avg=0.852120 VRAM=39.0GiB | 98.6% done | ETA(epoch): 311s + [Epoch 295][60/66] loss=0.871225 avg=0.855349 VRAM=38.9GiB | 98.6% done | ETA(epoch): 117s + Train loss: 0.851182 (1284.3s) ETA: 94min + Val loss: 0.832571 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9731 t_0.4-0.6=0.8750 t_0.6-0.8=0.7061 t_0.8-1.0=0.6148] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 24.4/188.5 GiB (12.9%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 296/299 (99% done) --- + [Epoch 296][10/66] loss=1.002504 avg=0.884161 VRAM=39.0GiB | 98.7% done | ETA(epoch): 1087s + [Epoch 296][20/66] loss=0.998803 avg=0.884003 VRAM=38.9GiB | 98.8% done | ETA(epoch): 893s + [Epoch 296][30/66] loss=0.782767 avg=0.880405 VRAM=39.0GiB | 98.8% done | ETA(epoch): 700s + [Epoch 296][40/66] loss=0.901958 avg=0.876580 VRAM=38.9GiB | 98.9% done | ETA(epoch): 506s + [Epoch 296][50/66] loss=0.894995 avg=0.868036 VRAM=39.0GiB | 98.9% done | ETA(epoch): 311s + [Epoch 296][60/66] loss=0.846586 avg=0.862796 VRAM=38.9GiB | 99.0% done | ETA(epoch): 117s + Train loss: 0.861878 (1284.0s) ETA: 71min + Val loss: 0.840661 [t_0.0-0.2=1.0754 t_0.2-0.4=1.0078 t_0.4-0.6=0.8596 t_0.6-0.8=0.6905 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 297/299 (99% done) --- + [Epoch 297][10/66] loss=0.830137 avg=0.856030 VRAM=39.0GiB | 99.1% done | ETA(epoch): 1094s + [Epoch 297][20/66] loss=0.884462 avg=0.845599 VRAM=38.9GiB | 99.1% done | ETA(epoch): 897s + [Epoch 297][30/66] loss=0.755129 avg=0.840607 VRAM=39.0GiB | 99.2% done | ETA(epoch): 702s + [Epoch 297][40/66] loss=0.952779 avg=0.849464 VRAM=38.9GiB | 99.2% done | ETA(epoch): 507s + [Epoch 297][50/66] loss=0.811750 avg=0.851903 VRAM=39.0GiB | 99.3% done | ETA(epoch): 312s + [Epoch 297][60/66] loss=0.832681 avg=0.849793 VRAM=38.9GiB | 99.3% done | ETA(epoch): 117s + Train loss: 0.852820 (1286.7s) ETA: 47min + Val loss: 0.823552 [t_0.0-0.2=1.0531 t_0.2-0.4=0.9731 t_0.4-0.6=0.8332 t_0.6-0.8=0.7323 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 298/299 (99% done) --- + [Epoch 298][10/66] loss=0.791350 avg=0.867913 VRAM=39.0GiB | 99.4% done | ETA(epoch): 1092s + [Epoch 298][20/66] loss=0.820386 avg=0.854343 VRAM=38.9GiB | 99.4% done | ETA(epoch): 896s + [Epoch 298][30/66] loss=0.901463 avg=0.859407 VRAM=39.0GiB | 99.5% done | ETA(epoch): 701s + [Epoch 298][40/66] loss=0.862118 avg=0.865378 VRAM=38.9GiB | 99.5% done | ETA(epoch): 507s + [Epoch 298][50/66] loss=0.896148 avg=0.867914 VRAM=39.0GiB | 99.6% done | ETA(epoch): 312s + [Epoch 298][60/66] loss=0.783828 avg=0.861651 VRAM=38.9GiB | 99.6% done | ETA(epoch): 117s + Train loss: 0.862214 (1287.7s) ETA: 24min + Val loss: 0.866909 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9793 t_0.4-0.6=0.8428 t_0.6-0.8=0.7312 t_0.8-1.0=0.6343] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.5/93.1 GiB (41.3%) + +--- Epoch 299/299 (100% done) --- + [Epoch 299][10/66] loss=0.902706 avg=0.790939 VRAM=39.0GiB | 99.7% done | ETA(epoch): 1090s + [Epoch 299][20/66] loss=0.912946 avg=0.827518 VRAM=38.9GiB | 99.8% done | ETA(epoch): 894s + [Epoch 299][30/66] loss=0.730195 avg=0.819887 VRAM=39.0GiB | 99.8% done | ETA(epoch): 706s + [Epoch 299][40/66] loss=0.926536 avg=0.832153 VRAM=38.9GiB | 99.9% done | ETA(epoch): 509s + [Epoch 299][50/66] loss=0.919646 avg=0.835919 VRAM=39.0GiB | 99.9% done | ETA(epoch): 313s + [Epoch 299][60/66] loss=0.708299 avg=0.840925 VRAM=38.9GiB | 100.0% done | ETA(epoch): 117s + [MilestoneVis] train_0 step 19799 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 19799 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 19799 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 19799 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 19799 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 19799 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 19799 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 19799 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 19799 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 19799 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_1024_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.840224 (2344.8s) ETA: 0min + Val loss: 0.807564 [t_0.0-0.2=1.0616 t_0.2-0.4=0.9731 t_0.4-0.6=0.8288 t_0.6-0.8=0.7083 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_1024_H100/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 23.5/188.5 GiB (12.4%) | VRAM: 38.5/93.1 GiB (41.3%) + [LossCurve WARN] could not save loss curve: No module named 'matplotlib' + +====================================================================== +Training complete. +====================================================================== +wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: uploading history steps 2510-2510, summary (0.1s) + m wandb: uploading history steps 2510-2510, summary (0.1s) + m wandb: uploading history steps 2510-2510, summary (0.1s) + m wandb: +wandb: Run summary: +wandb: adapter_lr 1e-05 +wandb: best_val_loss 0.78767 +wandb: cpu_pct 0.7 +wandb: epoch 299 +wandb: final_train_loss 0.84022 +wandb: flops/hdc2a_per_sample_gflops 1050.08808 +wandb: flops/total_per_sample_gflops 274712.2677 +wandb: flops/transformer_per_sample_gflops 273662.17962 +wandb: global_step 19793 +wandb: gpu_mem_gib 38.48744 +wandb: +10 ... +wandb: +wandb: View run lora_256_dim_1024_H100-train-20260520-185749 at: https://wandb.ai/hku-xg-boost/lora_256_dim_1024_H100/runs/sx2s3t28 +wandb: View project at: https://wandb.ai/hku-xg-boost/lora_256_dim_1024_H100 +wandb: Synced 4 W&B file(s), 110 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260520_185749-sx2s3t28/logs diff --git a/lora_256_dim_512_H100/step_vis/test_0_step_003300.png b/lora_256_dim_512_H100/step_vis/test_0_step_003300.png new file mode 100644 index 0000000000000000000000000000000000000000..5fd6157750f9a6b5b5c145a458d6634a239dd038 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_0_step_003300.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2f4820b2d3cd883ca9549c7565513f0141541939aab85167b3fb0cd02782c00 +size 1773727 diff --git a/lora_256_dim_512_H100/step_vis/test_0_step_006600_sample02.png b/lora_256_dim_512_H100/step_vis/test_0_step_006600_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..5437c206500401741f817037757aaaa7572a713d --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_0_step_006600_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:992103095a63e666e449a2dc3d2b5f34e96ac62b2c64c7fe000806279b8a04b0 +size 1836061 diff --git a/lora_256_dim_512_H100/step_vis/test_0_step_009900_sample00.png b/lora_256_dim_512_H100/step_vis/test_0_step_009900_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..a5f5fc126baef3ef678e7e61a7b93ebd54b23f5d --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_0_step_009900_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d128f7b4532cda3023960eea4162b4084af2e2b55c9a60d2bca46099579066 +size 1698993 diff --git a/lora_256_dim_512_H100/step_vis/test_0_step_009900_sample04.png b/lora_256_dim_512_H100/step_vis/test_0_step_009900_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..022355d55e7cfaceeaa4863ba6edc7f4f957a6bc --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_0_step_009900_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181f78d9ae96c976865b5c213449b91f7a007828c4cb1d0f4648931aef9821ce +size 1904216 diff --git a/lora_256_dim_512_H100/step_vis/test_1_step_013200.png b/lora_256_dim_512_H100/step_vis/test_1_step_013200.png new file mode 100644 index 0000000000000000000000000000000000000000..9ae25b8d37fecd213533c0c33fe5b3cdfb619017 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_1_step_013200.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a755dd6496ced393c83bdebb546be91aac0f4cc521c946e0061f110479b7a721 +size 1719178 diff --git a/lora_256_dim_512_H100/step_vis/test_1_step_019799_sample03.png b/lora_256_dim_512_H100/step_vis/test_1_step_019799_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..c57605cb4080178e71acca60b2f1c7d11cc1ffcb --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_1_step_019799_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6216e4d4602059073bb5a196d9e96926471d613e246f9642b0cc1327010307a2 +size 1678756 diff --git a/lora_256_dim_512_H100/step_vis/test_2_step_003300_sample04.png b/lora_256_dim_512_H100/step_vis/test_2_step_003300_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..1bb38419d9c8e1d9fd8e8cf462cccf330c5aa18f --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_2_step_003300_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32aa9314637342b1d8e79453806ce91f77dd590b9b7c9c0b9099fb7b3f71958b +size 1724648 diff --git a/lora_256_dim_512_H100/step_vis/test_2_step_006600_sample00.png b/lora_256_dim_512_H100/step_vis/test_2_step_006600_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..5324a9a7e190ff989693233d0baebd2af88ff32c --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_2_step_006600_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff5b27108a92967726ee830cb2628517d3c6351fc528b95370cf2a77c9ddcc63 +size 1808539 diff --git a/lora_256_dim_512_H100/step_vis/test_2_step_013200_sample00.png b/lora_256_dim_512_H100/step_vis/test_2_step_013200_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..ece6872b8721d192f69d2b51162a311189829cbb --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_2_step_013200_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ba7144dfc3db9d32248a9d27d354fee08cff200d003d85b35035b6ef28ed3cb +size 1805309 diff --git a/lora_256_dim_512_H100/step_vis/test_2_step_013200_sample04.png b/lora_256_dim_512_H100/step_vis/test_2_step_013200_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..85f088d08d0b0a66cec38a0c4357f9e75e32130d --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_2_step_013200_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d7d40bb529139739dfc721ba67911b2e11ab5e0c19ee8dbabdec693bebe8a1 +size 1793287 diff --git a/lora_256_dim_512_H100/step_vis/test_3_step_000000.png b/lora_256_dim_512_H100/step_vis/test_3_step_000000.png new file mode 100644 index 0000000000000000000000000000000000000000..87148af6c6aad995fc6df269060ffef66a1748a1 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_3_step_000000.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f7b77c07117cabe0d0d23a2be05bf83ad958c43ef794a35cfe63be3dc555d2 +size 2090786 diff --git a/lora_256_dim_512_H100/step_vis/test_3_step_003300_sample02.png b/lora_256_dim_512_H100/step_vis/test_3_step_003300_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..3a47c6c762117cd40021831f5d46645cd97d3fb4 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_3_step_003300_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e85629e8ee9172998f2a3b1b9668a2f74171295c6e1d63be5a0e9421c0a7f16 +size 1865284 diff --git a/lora_256_dim_512_H100/step_vis/test_3_step_013200_sample00.png b/lora_256_dim_512_H100/step_vis/test_3_step_013200_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..8d7df26c03acd87a7c334212f2c5ff52b5cd08a9 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_3_step_013200_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad1538916dad6d1ca9d5b68ca347bd85f0b76039eff63df9bc64a07b80c5cc80 +size 1756054 diff --git a/lora_256_dim_512_H100/step_vis/test_3_step_013200_sample01.png b/lora_256_dim_512_H100/step_vis/test_3_step_013200_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..412a6b053517b7cfa315c238c95e8e68bfd60bb9 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_3_step_013200_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39a0ce64df695d505f92be0b65737637b44334af5a4ca1fdbbf772d09b107f5 +size 1866521 diff --git a/lora_256_dim_512_H100/step_vis/test_3_step_019799_sample03.png b/lora_256_dim_512_H100/step_vis/test_3_step_019799_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..3fda999f009dc8708782c8ff285c98be949ec46c --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_3_step_019799_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccc038df1e011c24905371a7c6e4b0e1b07c5d502dcd731a7706b22b50d28849 +size 2146807 diff --git a/lora_256_dim_512_H100/step_vis/test_4_step_000000_sample04.png b/lora_256_dim_512_H100/step_vis/test_4_step_000000_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..199a406bd336480916e7b382c0a7fe646c668394 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_4_step_000000_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a8b9077798c7089164b34fada996cd49e7e456c4c4807f8790a5441480c1a0 +size 2764298 diff --git a/lora_256_dim_512_H100/step_vis/test_4_step_019799_sample01.png b/lora_256_dim_512_H100/step_vis/test_4_step_019799_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..c269aeb25b6d07264bfae9be0d0ebee0818194f8 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_4_step_019799_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23eb9740316f133e961537fcc62c57d022669036de4051d5c1508c5ae11a7569 +size 1911611 diff --git a/lora_256_dim_512_H100/step_vis/test_5_step_000000_sample03.png b/lora_256_dim_512_H100/step_vis/test_5_step_000000_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..96f4992bc83a525da7cb78bf923dae60ca867f4e --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_5_step_000000_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba0c1f70bbed7520b24c88dcb693624b52cc8d80b863f4dd40966294d0ad4ef1 +size 2712566 diff --git a/lora_256_dim_512_H100/step_vis/test_5_step_003300_sample00.png b/lora_256_dim_512_H100/step_vis/test_5_step_003300_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..4a11ca9d4ae063589e1afda72117a8335aa5134a --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_5_step_003300_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84ee85280c1c455d2fc17de67f51cc773f76f1e3f26d362342501cdfd78ef96e +size 1927513 diff --git a/lora_256_dim_512_H100/step_vis/test_5_step_016500_sample03.png b/lora_256_dim_512_H100/step_vis/test_5_step_016500_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..72d2381711e55d670fdb35222286ca43f6587bf4 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/test_5_step_016500_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df16e5e8751b5f083464eda01ae92f276e9d69ef7a5e9e759321af07cc2bdaa0 +size 1905776 diff --git a/lora_256_dim_512_H100/step_vis/train_0_step_000000_sample03.png b/lora_256_dim_512_H100/step_vis/train_0_step_000000_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..1399f459231561a9d4f95fe631e6f8a1a8b0d281 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_0_step_000000_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66bec9d71d711a4438cca2d1c7e927a4c81f4ada96bf82ba738b779b801a5cd6 +size 2751971 diff --git a/lora_256_dim_512_H100/step_vis/train_0_step_006600_sample01.png b/lora_256_dim_512_H100/step_vis/train_0_step_006600_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..512124fb76109d8ff3c1dde403fbea51aa6cbca5 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_0_step_006600_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6dbdae84b7c89bf74896287e991d43ec1c613448fe8713c3b39133f60afc02e +size 1771128 diff --git a/lora_256_dim_512_H100/step_vis/train_0_step_009900_sample02.png b/lora_256_dim_512_H100/step_vis/train_0_step_009900_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..ec9970f0f2fa307a4c6576bb4aea803ce089219c --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_0_step_009900_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7833772ebc303850a86e3c3080f4d49d8d8eff771a9835fa64a09cd32f3856ad +size 1869213 diff --git a/lora_256_dim_512_H100/step_vis/train_0_step_009900_sample03.png b/lora_256_dim_512_H100/step_vis/train_0_step_009900_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..43c8aa215a3e6751235c0191e97d46893d837e50 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_0_step_009900_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35382f3583044f2d1c2cc490fd25d37ca45cde0fd3a903859ed29298aac863bb +size 1658237 diff --git a/lora_256_dim_512_H100/step_vis/train_0_step_019799_sample00.png b/lora_256_dim_512_H100/step_vis/train_0_step_019799_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..d45be58317f236489a4482f4f2bec01cfc7fdb57 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_0_step_019799_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2079a5150feafd7442153a6ea2b3502fe400e8393104bc68c2a4ff6f7bc6bc34 +size 1717461 diff --git a/lora_256_dim_512_H100/step_vis/train_0_step_019799_sample01.png b/lora_256_dim_512_H100/step_vis/train_0_step_019799_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..a12a4caada63721da3da1f1ec089164619175ef4 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_0_step_019799_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abcdd9dc154d723e6921625ab3db60f4d0f1900216eeca1aaca85c47dafc7e00 +size 1713909 diff --git a/lora_256_dim_512_H100/step_vis/train_1_step_000000_sample03.png b/lora_256_dim_512_H100/step_vis/train_1_step_000000_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..3bb3383382b9206143e84d3aeb3599776e01186a --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_1_step_000000_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1624ad9172b04308344f80db6aacdc4d206e294c36678130b6707713642316b3 +size 2708260 diff --git a/lora_256_dim_512_H100/step_vis/train_1_step_019799.png b/lora_256_dim_512_H100/step_vis/train_1_step_019799.png new file mode 100644 index 0000000000000000000000000000000000000000..a436ee5423f0606f565010f601dff3cd6946a0db --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/train_1_step_019799.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8769af1e8cc2f04fc07685164f63842012edf6a016370a4b86c8e70232ae744 +size 1820103 diff --git a/lora_256_dim_512_H100/step_vis/val_0_step_013200_sample03.png b/lora_256_dim_512_H100/step_vis/val_0_step_013200_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..e02074b862123cd9bd59405711471443d7bdd7dc --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/val_0_step_013200_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1739dc7739f82b21d9bb0bf5d4ca7cfe52126aa35402ecfcfb75067dcaace7ae +size 1526580 diff --git a/lora_256_dim_512_H100/step_vis/val_0_step_019799_sample04.png b/lora_256_dim_512_H100/step_vis/val_0_step_019799_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..f8d1595af03635a696ec63b92a44f91f41c7fd54 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/val_0_step_019799_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34bdab3a7c64b65f177c4dd88c5ca6fcc6736405c4c79058272d4fa48322fcf7 +size 1782956 diff --git a/lora_256_dim_512_H100/step_vis/val_1_step_000000_sample03.png b/lora_256_dim_512_H100/step_vis/val_1_step_000000_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..1c7d84a14dc8daac0d4846fda0bdc8cbe76ff95b --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/val_1_step_000000_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa067b426669c10cab2706f68239f08389343dbc67589c164a9155623c39c6b8 +size 2767133 diff --git a/lora_256_dim_512_H100/step_vis/val_1_step_003300_sample03.png b/lora_256_dim_512_H100/step_vis/val_1_step_003300_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..da5a122d379c2e174800cc08fa4d0898eddc781a --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/val_1_step_003300_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691de381385b532b7002c495440d4cdabd592752876d40991ea8b2f4c323c996 +size 1798111 diff --git a/lora_256_dim_512_H100/step_vis/val_1_step_019799_sample04.png b/lora_256_dim_512_H100/step_vis/val_1_step_019799_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..546de6621f570f0115adf48e7a47685a5bb72fa3 --- /dev/null +++ b/lora_256_dim_512_H100/step_vis/val_1_step_019799_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:093f9e396df72b0851d14ab7a6f22a4dd679fe40e67971ddf20638edb25d011a +size 1742599 diff --git a/lora_256_dim_512_H100/train.log b/lora_256_dim_512_H100/train.log new file mode 100644 index 0000000000000000000000000000000000000000..087b2c0f77a2d0187fd3e9487818f09a37e68720 --- /dev/null +++ b/lora_256_dim_512_H100/train.log @@ -0,0 +1,4281 @@ + +====================================================================== +Command: train_script.py --name lora_256_dim_512_H100 --batch-size 6 --adapter-lr 1e-3 --lora-lr 1e-4 --adapter-mlp --lora-rank 256 --fusion-dim 512 --seed 42 +====================================================================== +Run dir : output/lora_256_dim_512_H100 +Log file: output/lora_256_dim_512_H100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260423_231404-v3sc1sdx +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_256_dim_512_H100-train-20260423-231404 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_256_dim_512_H100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_256_dim_512_H100/runs/v3sc1sdx + +Run Summary: + name lora_256_dim_512_H100 + fusion MLP (lightweight) + LoRA ON (rank=256) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_256_dim_512_H100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 512 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_256_dim_512_H100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.7/188.5 GiB (6.2%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.6s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 12.0/188.5 GiB (6.4%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (58.3s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 13.5/188.5 GiB (7.2%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 9.4M params +Control: 0.0M params +Total trainable: 9.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=256, alpha=256.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 78.64M + +Parameter Statistics: + HDC²A Adapter: total=9.4M trainable=9.4M + ControlNet (frozen): total=4212.2M LoRA trainable=78.64M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 9.4M + LoRA 78.64M = 88.09M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~8000 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0442, std=0.5195 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.430984 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.78 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.005310 + depth_encoder.conv_stem.6.weight: 0.004944 + W_s.weight: 0.003036 + semantic_encoder.conv_stem.3.weight: 0.002991 + semantic_encoder.proj.weight: 0.002960 + Test result: PASSED +[MEM @ after test] RAM: 14.0/188.5 GiB (7.4%) | VRAM: 38.1/93.1 GiB (40.9%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 672.097 GFLOPs + transformer: 273662.180 GFLOPs + total: 274334.277 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 500 epochs × 66 steps = 33000 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 3300, 6600, 9900, 13200, 16500, 19799, 23099, 26399, 29699, 32999] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/499 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/66] loss=0.996021 avg=0.923983 VRAM=39.0GiB | 0.0% done | ETA(epoch): 6725s + [Epoch 0][20/66] loss=0.950910 avg=0.917009 VRAM=38.8GiB | 0.1% done | ETA(epoch): 3206s + [Epoch 0][30/66] loss=1.057543 avg=0.919208 VRAM=39.0GiB | 0.1% done | ETA(epoch): 1902s + [Epoch 0][40/66] loss=0.860612 avg=0.917848 VRAM=38.8GiB | 0.1% done | ETA(epoch): 1155s + [Epoch 0][50/66] loss=0.902975 avg=0.920852 VRAM=39.0GiB | 0.2% done | ETA(epoch): 630s + [Epoch 0][60/66] loss=0.951722 avg=0.918388 VRAM=38.8GiB | 0.2% done | ETA(epoch): 216s + Train loss: 0.919216 (2274.4s) ETA: 18953min + Val loss: 1.020531 [t_0.0-0.2=1.0888 t_0.2-0.4=1.0667 t_0.4-0.6=0.8857 t_0.6-0.8=0.7746 t_0.8-1.0=1.1116] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 1/499 (0% done) --- + [Epoch 1][10/66] loss=1.020646 avg=0.905561 VRAM=39.0GiB | 0.2% done | ETA(epoch): 1170s + [Epoch 1][20/66] loss=0.917925 avg=0.907168 VRAM=38.8GiB | 0.3% done | ETA(epoch): 920s + [Epoch 1][30/66] loss=0.968527 avg=0.910933 VRAM=39.0GiB | 0.3% done | ETA(epoch): 710s + [Epoch 1][40/66] loss=0.993664 avg=0.913719 VRAM=38.8GiB | 0.3% done | ETA(epoch): 509s + [Epoch 1][50/66] loss=0.870458 avg=0.918239 VRAM=39.0GiB | 0.4% done | ETA(epoch): 312s + [Epoch 1][60/66] loss=0.903457 avg=0.918072 VRAM=38.8GiB | 0.4% done | ETA(epoch): 117s + Train loss: 0.917160 (1280.6s) ETA: 15163min + Val loss: 0.908054 [t_0.0-0.2=1.0841 t_0.2-0.4=1.0273 t_0.4-0.6=0.8776 t_0.6-0.8=0.7673 t_0.8-1.0=0.9046] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 2/499 (0% done) --- + [Epoch 2][10/66] loss=0.931147 avg=0.911352 VRAM=39.0GiB | 0.4% done | ETA(epoch): 1070s + [Epoch 2][20/66] loss=0.938794 avg=0.915478 VRAM=38.8GiB | 0.5% done | ETA(epoch): 880s + [Epoch 2][30/66] loss=0.856924 avg=0.913370 VRAM=39.0GiB | 0.5% done | ETA(epoch): 689s + [Epoch 2][40/66] loss=0.883569 avg=0.917996 VRAM=38.8GiB | 0.5% done | ETA(epoch): 498s + [Epoch 2][50/66] loss=0.848230 avg=0.911157 VRAM=39.0GiB | 0.6% done | ETA(epoch): 306s + [Epoch 2][60/66] loss=0.855815 avg=0.905246 VRAM=38.8GiB | 0.6% done | ETA(epoch): 115s + Train loss: 0.905236 (1264.2s) ETA: 13859min + Val loss: 0.950950 [t_0.0-0.2=1.0971 t_0.2-0.4=1.0214 t_0.4-0.6=0.8328 t_0.6-0.8=0.7581 t_0.8-1.0=0.9908] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 3/499 (1% done) --- + [Epoch 3][10/66] loss=0.829429 avg=0.924085 VRAM=39.0GiB | 0.6% done | ETA(epoch): 1073s + [Epoch 3][20/66] loss=0.881167 avg=0.914141 VRAM=38.8GiB | 0.7% done | ETA(epoch): 883s + [Epoch 3][30/66] loss=0.905954 avg=0.907854 VRAM=39.0GiB | 0.7% done | ETA(epoch): 691s + [Epoch 3][40/66] loss=0.973570 avg=0.907615 VRAM=38.8GiB | 0.7% done | ETA(epoch): 499s + [Epoch 3][50/66] loss=0.870821 avg=0.903702 VRAM=39.0GiB | 0.8% done | ETA(epoch): 307s + [Epoch 3][60/66] loss=0.857071 avg=0.900882 VRAM=38.8GiB | 0.8% done | ETA(epoch): 115s + Train loss: 0.898488 (1265.1s) ETA: 13182min + Val loss: 0.888543 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9775 t_0.4-0.6=0.8575 t_0.6-0.8=0.7735 t_0.8-1.0=0.9326] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 4/499 (1% done) --- + [Epoch 4][10/66] loss=0.887449 avg=0.867496 VRAM=39.0GiB | 0.8% done | ETA(epoch): 1075s + [Epoch 4][20/66] loss=0.876555 avg=0.882984 VRAM=38.8GiB | 0.9% done | ETA(epoch): 882s + [Epoch 4][30/66] loss=0.890715 avg=0.881822 VRAM=39.0GiB | 0.9% done | ETA(epoch): 690s + [Epoch 4][40/66] loss=0.865174 avg=0.875283 VRAM=38.8GiB | 0.9% done | ETA(epoch): 498s + [Epoch 4][50/66] loss=0.894648 avg=0.884430 VRAM=39.0GiB | 1.0% done | ETA(epoch): 308s + [Epoch 4][60/66] loss=0.896396 avg=0.875710 VRAM=38.8GiB | 1.0% done | ETA(epoch): 115s + Train loss: 0.875862 (1267.7s) ETA: 12772min + Val loss: 0.936644 [t_0.0-0.2=1.0821 t_0.2-0.4=1.0147 t_0.4-0.6=0.8809 t_0.6-0.8=0.7369 t_0.8-1.0=0.9287] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 5/499 (1% done) --- + [Epoch 5][10/66] loss=0.822586 avg=0.879532 VRAM=39.0GiB | 1.0% done | ETA(epoch): 1071s + [Epoch 5][20/66] loss=0.898609 avg=0.880502 VRAM=38.8GiB | 1.1% done | ETA(epoch): 880s + [Epoch 5][30/66] loss=0.860028 avg=0.880420 VRAM=39.0GiB | 1.1% done | ETA(epoch): 689s + [Epoch 5][40/66] loss=0.986131 avg=0.891687 VRAM=38.8GiB | 1.1% done | ETA(epoch): 510s + [Epoch 5][50/66] loss=0.844808 avg=0.886074 VRAM=39.0GiB | 1.2% done | ETA(epoch): 312s + [Epoch 5][60/66] loss=0.826761 avg=0.883151 VRAM=38.8GiB | 1.2% done | ETA(epoch): 117s + Train loss: 0.884270 (1282.5s) ETA: 12511min + Val loss: 0.892923 [t_0.0-0.2=1.0857 t_0.2-0.4=0.9819 t_0.4-0.6=0.8356 t_0.6-0.8=0.7661 t_0.8-1.0=0.8510] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 6/499 (1% done) --- + [Epoch 6][10/66] loss=0.831961 avg=0.894636 VRAM=39.0GiB | 1.2% done | ETA(epoch): 1072s + [Epoch 6][20/66] loss=0.921910 avg=0.894761 VRAM=38.8GiB | 1.3% done | ETA(epoch): 880s + [Epoch 6][30/66] loss=0.883422 avg=0.893150 VRAM=39.0GiB | 1.3% done | ETA(epoch): 689s + [Epoch 6][40/66] loss=0.940795 avg=0.895579 VRAM=38.8GiB | 1.3% done | ETA(epoch): 498s + [Epoch 6][50/66] loss=0.883447 avg=0.899723 VRAM=39.0GiB | 1.4% done | ETA(epoch): 306s + [Epoch 6][60/66] loss=0.899043 avg=0.894020 VRAM=38.8GiB | 1.4% done | ETA(epoch): 115s + Train loss: 0.893988 (1263.6s) ETA: 12296min + Val loss: 0.919961 [t_0.0-0.2=1.0690 t_0.2-0.4=1.0148 t_0.4-0.6=0.8417 t_0.6-0.8=0.7589 t_0.8-1.0=0.9291] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0006 +[MEM @ epoch 6 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 7/499 (1% done) --- + [Epoch 7][10/66] loss=0.945812 avg=0.919197 VRAM=39.0GiB | 1.4% done | ETA(epoch): 1071s + [Epoch 7][20/66] loss=0.824102 avg=0.896178 VRAM=38.8GiB | 1.5% done | ETA(epoch): 880s + [Epoch 7][30/66] loss=0.857899 avg=0.898495 VRAM=39.0GiB | 1.5% done | ETA(epoch): 689s + [Epoch 7][40/66] loss=0.860950 avg=0.895107 VRAM=38.8GiB | 1.5% done | ETA(epoch): 497s + [Epoch 7][50/66] loss=0.846639 avg=0.887980 VRAM=39.0GiB | 1.6% done | ETA(epoch): 306s + [Epoch 7][60/66] loss=0.900606 avg=0.890836 VRAM=38.8GiB | 1.6% done | ETA(epoch): 115s + Train loss: 0.892324 (1262.3s) ETA: 12128min + Val loss: 0.862610 [t_0.0-0.2=1.0859 t_0.2-0.4=0.9808 t_0.4-0.6=0.8656 t_0.6-0.8=0.7502 t_0.8-1.0=0.7632] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0007 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 8/499 (2% done) --- + [Epoch 8][10/66] loss=0.845001 avg=0.861295 VRAM=39.0GiB | 1.6% done | ETA(epoch): 1072s + [Epoch 8][20/66] loss=0.924297 avg=0.870855 VRAM=38.8GiB | 1.7% done | ETA(epoch): 884s + [Epoch 8][30/66] loss=0.905143 avg=0.872508 VRAM=39.0GiB | 1.7% done | ETA(epoch): 691s + [Epoch 8][40/66] loss=0.926068 avg=0.882110 VRAM=38.8GiB | 1.7% done | ETA(epoch): 499s + [Epoch 8][50/66] loss=0.721362 avg=0.879772 VRAM=39.0GiB | 1.8% done | ETA(epoch): 307s + [Epoch 8][60/66] loss=0.801751 avg=0.879207 VRAM=38.8GiB | 1.8% done | ETA(epoch): 115s + Train loss: 0.876615 (1264.5s) ETA: 11994min + Val loss: 0.879127 [t_0.0-0.2=1.0810 t_0.2-0.4=1.0236 t_0.4-0.6=0.8607 t_0.6-0.8=0.7371 t_0.8-1.0=0.7688] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 16.1/188.5 GiB (8.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 9/499 (2% done) --- + [Epoch 9][10/66] loss=0.928527 avg=0.880146 VRAM=39.0GiB | 1.8% done | ETA(epoch): 1071s + [Epoch 9][20/66] loss=0.834448 avg=0.872435 VRAM=38.8GiB | 1.9% done | ETA(epoch): 880s + [Epoch 9][30/66] loss=0.865812 avg=0.875920 VRAM=39.0GiB | 1.9% done | ETA(epoch): 689s + [Epoch 9][40/66] loss=1.046726 avg=0.878207 VRAM=38.8GiB | 1.9% done | ETA(epoch): 498s + [Epoch 9][50/66] loss=0.743672 avg=0.873505 VRAM=39.0GiB | 2.0% done | ETA(epoch): 306s + [Epoch 9][60/66] loss=0.844119 avg=0.874398 VRAM=38.8GiB | 2.0% done | ETA(epoch): 115s + Train loss: 0.880499 (1264.1s) ETA: 11883min + Val loss: 0.888400 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0221 t_0.4-0.6=0.8586 t_0.6-0.8=0.7197 t_0.8-1.0=0.7422] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 10/499 (2% done) --- + [Epoch 10][10/66] loss=0.820841 avg=0.887861 VRAM=39.0GiB | 2.0% done | ETA(epoch): 1071s + [Epoch 10][20/66] loss=0.813798 avg=0.877345 VRAM=38.8GiB | 2.1% done | ETA(epoch): 880s + [Epoch 10][30/66] loss=0.974298 avg=0.872086 VRAM=39.0GiB | 2.1% done | ETA(epoch): 689s + [Epoch 10][40/66] loss=0.870616 avg=0.876560 VRAM=38.8GiB | 2.1% done | ETA(epoch): 497s + [Epoch 10][50/66] loss=0.848396 avg=0.874737 VRAM=39.0GiB | 2.2% done | ETA(epoch): 306s + [Epoch 10][60/66] loss=0.853350 avg=0.874650 VRAM=38.8GiB | 2.2% done | ETA(epoch): 115s + Train loss: 0.875195 (1264.5s) ETA: 11790min + Val loss: 0.886706 [t_0.0-0.2=1.0771 t_0.2-0.4=1.0191 t_0.4-0.6=0.8451 t_0.6-0.8=0.7239 t_0.8-1.0=0.7737] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0010 +[MEM @ epoch 10 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 11/499 (2% done) --- + [Epoch 11][10/66] loss=0.769477 avg=0.869411 VRAM=39.0GiB | 2.2% done | ETA(epoch): 1071s + [Epoch 11][20/66] loss=0.936199 avg=0.875300 VRAM=38.8GiB | 2.3% done | ETA(epoch): 880s + [Epoch 11][30/66] loss=0.820394 avg=0.873916 VRAM=39.0GiB | 2.3% done | ETA(epoch): 689s + [Epoch 11][40/66] loss=0.815252 avg=0.870492 VRAM=38.8GiB | 2.3% done | ETA(epoch): 497s + [Epoch 11][50/66] loss=0.934335 avg=0.868272 VRAM=39.0GiB | 2.4% done | ETA(epoch): 306s + [Epoch 11][60/66] loss=0.862769 avg=0.865124 VRAM=38.8GiB | 2.4% done | ETA(epoch): 115s + Train loss: 0.864720 (1264.4s) ETA: 11707min + Val loss: 0.882625 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9960 t_0.4-0.6=0.8308 t_0.6-0.8=0.7472 t_0.8-1.0=0.7555] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 12/499 (2% done) --- + [Epoch 12][10/66] loss=0.886075 avg=0.883815 VRAM=39.0GiB | 2.4% done | ETA(epoch): 1071s + [Epoch 12][20/66] loss=0.919132 avg=0.908542 VRAM=38.8GiB | 2.5% done | ETA(epoch): 881s + [Epoch 12][30/66] loss=0.889749 avg=0.900780 VRAM=39.0GiB | 2.5% done | ETA(epoch): 689s + [Epoch 12][40/66] loss=0.896498 avg=0.892164 VRAM=38.8GiB | 2.5% done | ETA(epoch): 498s + [Epoch 12][50/66] loss=0.807658 avg=0.888191 VRAM=39.0GiB | 2.6% done | ETA(epoch): 306s + [Epoch 12][60/66] loss=0.849328 avg=0.887203 VRAM=38.8GiB | 2.6% done | ETA(epoch): 115s + Train loss: 0.884152 (1263.3s) ETA: 11632min + Val loss: 0.877134 [t_0.0-0.2=1.0775 t_0.2-0.4=1.0153 t_0.4-0.6=0.8601 t_0.6-0.8=0.7279 t_0.8-1.0=0.7370] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 13/499 (3% done) --- + [Epoch 13][10/66] loss=0.935314 avg=0.894784 VRAM=39.0GiB | 2.6% done | ETA(epoch): 1071s + [Epoch 13][20/66] loss=0.795491 avg=0.892280 VRAM=38.8GiB | 2.7% done | ETA(epoch): 879s + [Epoch 13][30/66] loss=0.867093 avg=0.896254 VRAM=39.0GiB | 2.7% done | ETA(epoch): 688s + [Epoch 13][40/66] loss=0.926642 avg=0.895744 VRAM=38.8GiB | 2.7% done | ETA(epoch): 498s + [Epoch 13][50/66] loss=0.812497 avg=0.889685 VRAM=39.0GiB | 2.8% done | ETA(epoch): 307s + [Epoch 13][60/66] loss=0.970946 avg=0.886760 VRAM=38.8GiB | 2.8% done | ETA(epoch): 115s + Train loss: 0.885308 (1264.1s) ETA: 11565min + Val loss: 0.864957 [t_0.0-0.2=1.0837 t_0.2-0.4=1.0108 t_0.4-0.6=0.8496 t_0.6-0.8=0.7376 t_0.8-1.0=0.7264] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 14/499 (3% done) --- + [Epoch 14][10/66] loss=0.888987 avg=0.882549 VRAM=39.0GiB | 2.8% done | ETA(epoch): 1072s + [Epoch 14][20/66] loss=0.898243 avg=0.906341 VRAM=38.8GiB | 2.9% done | ETA(epoch): 881s + [Epoch 14][30/66] loss=0.925892 avg=0.896000 VRAM=39.0GiB | 2.9% done | ETA(epoch): 689s + [Epoch 14][40/66] loss=0.917101 avg=0.889051 VRAM=38.8GiB | 2.9% done | ETA(epoch): 498s + [Epoch 14][50/66] loss=0.826155 avg=0.879490 VRAM=39.0GiB | 3.0% done | ETA(epoch): 306s + [Epoch 14][60/66] loss=0.908292 avg=0.879427 VRAM=38.8GiB | 3.0% done | ETA(epoch): 115s + Train loss: 0.878510 (1263.4s) ETA: 11503min + Val loss: 0.873870 [t_0.0-0.2=1.0735 t_0.2-0.4=1.0149 t_0.4-0.6=0.8844 t_0.6-0.8=0.7000 t_0.8-1.0=0.7219] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0014 + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 15/499 (3% done) --- + [Epoch 15][10/66] loss=0.835002 avg=0.869766 VRAM=39.0GiB | 3.0% done | ETA(epoch): 1071s + [Epoch 15][20/66] loss=0.805410 avg=0.874988 VRAM=38.8GiB | 3.1% done | ETA(epoch): 880s + [Epoch 15][30/66] loss=0.849533 avg=0.873193 VRAM=39.0GiB | 3.1% done | ETA(epoch): 693s + [Epoch 15][40/66] loss=0.819443 avg=0.872240 VRAM=38.8GiB | 3.1% done | ETA(epoch): 500s + [Epoch 15][50/66] loss=0.987436 avg=0.875349 VRAM=39.0GiB | 3.2% done | ETA(epoch): 307s + [Epoch 15][60/66] loss=0.847804 avg=0.870160 VRAM=38.8GiB | 3.2% done | ETA(epoch): 115s + Train loss: 0.871074 (1267.4s) ETA: 11449min + Val loss: 0.876153 [t_0.0-0.2=1.0698 t_0.2-0.4=1.0147 t_0.4-0.6=0.8544 t_0.6-0.8=0.6980 t_0.8-1.0=0.7411] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 16/499 (3% done) --- + [Epoch 16][10/66] loss=0.773860 avg=0.890506 VRAM=39.0GiB | 3.2% done | ETA(epoch): 1070s + [Epoch 16][20/66] loss=0.935649 avg=0.887989 VRAM=38.8GiB | 3.3% done | ETA(epoch): 880s + [Epoch 16][30/66] loss=0.927338 avg=0.885840 VRAM=39.0GiB | 3.3% done | ETA(epoch): 688s + [Epoch 16][40/66] loss=0.987658 avg=0.884243 VRAM=38.8GiB | 3.3% done | ETA(epoch): 497s + [Epoch 16][50/66] loss=0.885801 avg=0.886162 VRAM=39.0GiB | 3.4% done | ETA(epoch): 306s + [Epoch 16][60/66] loss=0.797954 avg=0.883605 VRAM=38.8GiB | 3.4% done | ETA(epoch): 115s + Train loss: 0.884113 (1262.3s) ETA: 11396min + Val loss: 0.882723 [t_0.0-0.2=1.0802 t_0.2-0.4=0.9895 t_0.4-0.6=0.8622 t_0.6-0.8=0.7315 t_0.8-1.0=0.7148] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 17/499 (3% done) --- + [Epoch 17][10/66] loss=0.885306 avg=0.860476 VRAM=39.0GiB | 3.4% done | ETA(epoch): 1082s + [Epoch 17][20/66] loss=0.894407 avg=0.860264 VRAM=38.8GiB | 3.5% done | ETA(epoch): 884s + [Epoch 17][30/66] loss=0.859583 avg=0.863457 VRAM=39.0GiB | 3.5% done | ETA(epoch): 691s + [Epoch 17][40/66] loss=0.718257 avg=0.869909 VRAM=38.8GiB | 3.5% done | ETA(epoch): 499s + [Epoch 17][50/66] loss=0.931165 avg=0.867033 VRAM=39.0GiB | 3.6% done | ETA(epoch): 307s + [Epoch 17][60/66] loss=0.998692 avg=0.873003 VRAM=38.8GiB | 3.6% done | ETA(epoch): 115s + Train loss: 0.873129 (1264.4s) ETA: 11347min + Val loss: 0.886121 [t_0.0-0.2=1.0755 t_0.2-0.4=1.0134 t_0.4-0.6=0.8716 t_0.6-0.8=0.7421 t_0.8-1.0=0.7236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 18/499 (4% done) --- + [Epoch 18][10/66] loss=0.861395 avg=0.843632 VRAM=39.0GiB | 3.6% done | ETA(epoch): 1072s + [Epoch 18][20/66] loss=0.836577 avg=0.862150 VRAM=38.8GiB | 3.7% done | ETA(epoch): 880s + [Epoch 18][30/66] loss=0.872706 avg=0.870177 VRAM=39.0GiB | 3.7% done | ETA(epoch): 689s + [Epoch 18][40/66] loss=0.833872 avg=0.866109 VRAM=38.8GiB | 3.7% done | ETA(epoch): 498s + [Epoch 18][50/66] loss=0.856256 avg=0.861549 VRAM=39.0GiB | 3.8% done | ETA(epoch): 306s + [Epoch 18][60/66] loss=0.905643 avg=0.855805 VRAM=38.8GiB | 3.8% done | ETA(epoch): 115s + Train loss: 0.857104 (1265.7s) ETA: 11301min + Val loss: 0.907800 [t_0.0-0.2=1.0844 t_0.2-0.4=1.0435 t_0.4-0.6=0.8527 t_0.6-0.8=0.7205 t_0.8-1.0=0.6953] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 19/499 (4% done) --- + [Epoch 19][10/66] loss=0.936056 avg=0.876013 VRAM=39.0GiB | 3.8% done | ETA(epoch): 1071s + [Epoch 19][20/66] loss=0.846451 avg=0.883054 VRAM=38.8GiB | 3.9% done | ETA(epoch): 880s + [Epoch 19][30/66] loss=0.961462 avg=0.884518 VRAM=39.0GiB | 3.9% done | ETA(epoch): 688s + [Epoch 19][40/66] loss=0.932851 avg=0.889316 VRAM=38.8GiB | 3.9% done | ETA(epoch): 497s + [Epoch 19][50/66] loss=0.870814 avg=0.887712 VRAM=39.0GiB | 4.0% done | ETA(epoch): 306s + [Epoch 19][60/66] loss=0.871505 avg=0.882439 VRAM=38.8GiB | 4.0% done | ETA(epoch): 115s + Train loss: 0.885273 (1266.5s) ETA: 11258min + Val loss: 0.885752 [t_0.0-0.2=1.0615 t_0.2-0.4=1.0213 t_0.4-0.6=0.8563 t_0.6-0.8=0.7470 t_0.8-1.0=0.6989] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 16.1/188.5 GiB (8.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 20/499 (4% done) --- + [Epoch 20][10/66] loss=0.787963 avg=0.847587 VRAM=39.0GiB | 4.0% done | ETA(epoch): 1071s + [Epoch 20][20/66] loss=0.951877 avg=0.855662 VRAM=38.8GiB | 4.1% done | ETA(epoch): 880s + [Epoch 20][30/66] loss=0.887752 avg=0.857911 VRAM=39.0GiB | 4.1% done | ETA(epoch): 689s + [Epoch 20][40/66] loss=0.760774 avg=0.855445 VRAM=38.8GiB | 4.1% done | ETA(epoch): 498s + [Epoch 20][50/66] loss=0.876037 avg=0.865106 VRAM=39.0GiB | 4.2% done | ETA(epoch): 307s + [Epoch 20][60/66] loss=0.819870 avg=0.864722 VRAM=38.8GiB | 4.2% done | ETA(epoch): 115s + Train loss: 0.865728 (1265.5s) ETA: 11217min + Val loss: 0.884420 [t_0.0-0.2=1.0783 t_0.2-0.4=1.0062 t_0.4-0.6=0.8508 t_0.6-0.8=0.7512 t_0.8-1.0=0.7015] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 21/499 (4% done) --- + [Epoch 21][10/66] loss=0.808429 avg=0.858292 VRAM=39.0GiB | 4.2% done | ETA(epoch): 1071s + [Epoch 21][20/66] loss=0.822795 avg=0.845044 VRAM=38.8GiB | 4.3% done | ETA(epoch): 880s + [Epoch 21][30/66] loss=0.842559 avg=0.852083 VRAM=39.0GiB | 4.3% done | ETA(epoch): 689s + [Epoch 21][40/66] loss=0.758970 avg=0.845606 VRAM=38.8GiB | 4.3% done | ETA(epoch): 498s + [Epoch 21][50/66] loss=0.822230 avg=0.850220 VRAM=39.0GiB | 4.4% done | ETA(epoch): 306s + [Epoch 21][60/66] loss=0.910191 avg=0.851562 VRAM=38.8GiB | 4.4% done | ETA(epoch): 115s + Train loss: 0.849782 (1263.4s) ETA: 11176min + Val loss: 0.875018 [t_0.0-0.2=1.0641 t_0.2-0.4=0.9842 t_0.4-0.6=0.8720 t_0.6-0.8=0.7182 t_0.8-1.0=0.6916] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 16.1/188.5 GiB (8.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 22/499 (4% done) --- + [Epoch 22][10/66] loss=0.825677 avg=0.828305 VRAM=39.0GiB | 4.4% done | ETA(epoch): 1072s + [Epoch 22][20/66] loss=0.938172 avg=0.858865 VRAM=38.8GiB | 4.5% done | ETA(epoch): 881s + [Epoch 22][30/66] loss=0.789813 avg=0.848219 VRAM=39.0GiB | 4.5% done | ETA(epoch): 689s + [Epoch 22][40/66] loss=0.874797 avg=0.856413 VRAM=38.8GiB | 4.5% done | ETA(epoch): 498s + [Epoch 22][50/66] loss=0.981055 avg=0.861510 VRAM=39.0GiB | 4.6% done | ETA(epoch): 306s + [Epoch 22][60/66] loss=0.871065 avg=0.860528 VRAM=38.8GiB | 4.6% done | ETA(epoch): 115s + Train loss: 0.864349 (1263.4s) ETA: 11137min + Val loss: 0.877237 [t_0.0-0.2=1.0879 t_0.2-0.4=1.0071 t_0.4-0.6=0.8287 t_0.6-0.8=0.7415 t_0.8-1.0=0.6889] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 23/499 (5% done) --- + [Epoch 23][10/66] loss=0.874227 avg=0.875413 VRAM=39.0GiB | 4.6% done | ETA(epoch): 1071s + [Epoch 23][20/66] loss=0.881190 avg=0.864481 VRAM=38.8GiB | 4.7% done | ETA(epoch): 881s + [Epoch 23][30/66] loss=0.815626 avg=0.858957 VRAM=39.0GiB | 4.7% done | ETA(epoch): 689s + [Epoch 23][40/66] loss=0.829309 avg=0.857657 VRAM=38.8GiB | 4.7% done | ETA(epoch): 498s + [Epoch 23][50/66] loss=0.879701 avg=0.862270 VRAM=39.0GiB | 4.8% done | ETA(epoch): 306s + [Epoch 23][60/66] loss=0.943375 avg=0.864318 VRAM=38.8GiB | 4.8% done | ETA(epoch): 115s + Train loss: 0.870203 (1263.4s) ETA: 11100min + Val loss: 0.897116 [t_0.0-0.2=1.0653 t_0.2-0.4=1.0046 t_0.4-0.6=0.8441 t_0.6-0.8=0.7408 t_0.8-1.0=0.6659] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 24/499 (5% done) --- + [Epoch 24][10/66] loss=0.889272 avg=0.874432 VRAM=39.0GiB | 4.8% done | ETA(epoch): 1071s + [Epoch 24][20/66] loss=0.853659 avg=0.873777 VRAM=38.8GiB | 4.9% done | ETA(epoch): 880s + [Epoch 24][30/66] loss=0.775510 avg=0.861660 VRAM=39.0GiB | 4.9% done | ETA(epoch): 689s + [Epoch 24][40/66] loss=0.788175 avg=0.861960 VRAM=38.8GiB | 4.9% done | ETA(epoch): 497s + [Epoch 24][50/66] loss=0.853578 avg=0.864253 VRAM=39.0GiB | 5.0% done | ETA(epoch): 306s + [Epoch 24][60/66] loss=0.812202 avg=0.868051 VRAM=38.8GiB | 5.0% done | ETA(epoch): 115s + Train loss: 0.867448 (1263.1s) ETA: 11063min + Val loss: 0.885053 [t_0.0-0.2=1.0895 t_0.2-0.4=0.9795 t_0.4-0.6=0.8298 t_0.6-0.8=0.7214 t_0.8-1.0=0.6844] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 25/499 (5% done) --- + [Epoch 25][10/66] loss=0.875362 avg=0.863056 VRAM=39.0GiB | 5.0% done | ETA(epoch): 1071s + [Epoch 25][20/66] loss=0.848353 avg=0.855413 VRAM=38.8GiB | 5.1% done | ETA(epoch): 881s + [Epoch 25][30/66] loss=0.849847 avg=0.861600 VRAM=39.0GiB | 5.1% done | ETA(epoch): 689s + [Epoch 25][40/66] loss=0.860909 avg=0.871254 VRAM=38.8GiB | 5.1% done | ETA(epoch): 498s + [Epoch 25][50/66] loss=0.797136 avg=0.867400 VRAM=39.0GiB | 5.2% done | ETA(epoch): 306s + [Epoch 25][60/66] loss=0.719204 avg=0.859371 VRAM=38.8GiB | 5.2% done | ETA(epoch): 115s + Train loss: 0.855830 (1263.3s) ETA: 11028min + Val loss: 0.871678 [t_0.0-0.2=1.0699 t_0.2-0.4=1.0136 t_0.4-0.6=0.8839 t_0.6-0.8=0.7223 t_0.8-1.0=0.6923] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 26/499 (5% done) --- + [Epoch 26][10/66] loss=0.870632 avg=0.884822 VRAM=39.0GiB | 5.2% done | ETA(epoch): 1071s + [Epoch 26][20/66] loss=0.934637 avg=0.897205 VRAM=38.8GiB | 5.3% done | ETA(epoch): 880s + [Epoch 26][30/66] loss=0.748666 avg=0.872789 VRAM=39.0GiB | 5.3% done | ETA(epoch): 689s + [Epoch 26][40/66] loss=0.845170 avg=0.868354 VRAM=38.8GiB | 5.3% done | ETA(epoch): 497s + [Epoch 26][50/66] loss=0.874739 avg=0.872598 VRAM=39.0GiB | 5.4% done | ETA(epoch): 306s + [Epoch 26][60/66] loss=0.801744 avg=0.869159 VRAM=38.8GiB | 5.4% done | ETA(epoch): 115s + Train loss: 0.864204 (1264.6s) ETA: 10994min + Val loss: 0.871840 [t_0.0-0.2=1.0684 t_0.2-0.4=1.0209 t_0.4-0.6=0.8650 t_0.6-0.8=0.7208 t_0.8-1.0=0.6702] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 27/499 (5% done) --- + [Epoch 27][10/66] loss=0.881987 avg=0.898796 VRAM=39.0GiB | 5.4% done | ETA(epoch): 1071s + [Epoch 27][20/66] loss=0.786324 avg=0.875591 VRAM=38.8GiB | 5.5% done | ETA(epoch): 880s + [Epoch 27][30/66] loss=0.884335 avg=0.869805 VRAM=39.0GiB | 5.5% done | ETA(epoch): 689s + [Epoch 27][40/66] loss=0.902559 avg=0.872600 VRAM=38.8GiB | 5.5% done | ETA(epoch): 498s + [Epoch 27][50/66] loss=0.761036 avg=0.869422 VRAM=39.0GiB | 5.6% done | ETA(epoch): 306s + [Epoch 27][60/66] loss=0.971750 avg=0.867104 VRAM=38.8GiB | 5.6% done | ETA(epoch): 115s + Train loss: 0.868153 (1265.8s) ETA: 10961min + Val loss: 0.862867 [t_0.0-0.2=1.0684 t_0.2-0.4=1.0133 t_0.4-0.6=0.8317 t_0.6-0.8=0.7276 t_0.8-1.0=0.6872] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 28/499 (6% done) --- + [Epoch 28][10/66] loss=1.019171 avg=0.900095 VRAM=39.0GiB | 5.6% done | ETA(epoch): 1071s + [Epoch 28][20/66] loss=0.944710 avg=0.892592 VRAM=38.8GiB | 5.7% done | ETA(epoch): 880s + [Epoch 28][30/66] loss=0.796252 avg=0.874497 VRAM=39.0GiB | 5.7% done | ETA(epoch): 688s + [Epoch 28][40/66] loss=0.932686 avg=0.861609 VRAM=38.8GiB | 5.7% done | ETA(epoch): 497s + [Epoch 28][50/66] loss=0.737747 avg=0.864482 VRAM=39.0GiB | 5.8% done | ETA(epoch): 306s + [Epoch 28][60/66] loss=0.942485 avg=0.874983 VRAM=38.8GiB | 5.8% done | ETA(epoch): 115s + Train loss: 0.877560 (1261.7s) ETA: 10928min + Val loss: 0.854231 [t_0.0-0.2=1.0884 t_0.2-0.4=0.9523 t_0.4-0.6=0.8387 t_0.6-0.8=0.7294 t_0.8-1.0=0.6952] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0028 (BEST) + Deleted old checkpoint: checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 29/499 (6% done) --- + [Epoch 29][10/66] loss=0.876869 avg=0.831906 VRAM=39.0GiB | 5.8% done | ETA(epoch): 1068s + [Epoch 29][20/66] loss=0.794518 avg=0.845142 VRAM=38.8GiB | 5.9% done | ETA(epoch): 879s + [Epoch 29][30/66] loss=0.885471 avg=0.870056 VRAM=39.0GiB | 5.9% done | ETA(epoch): 688s + [Epoch 29][40/66] loss=0.859216 avg=0.863792 VRAM=38.8GiB | 5.9% done | ETA(epoch): 498s + [Epoch 29][50/66] loss=0.899872 avg=0.870523 VRAM=39.0GiB | 6.0% done | ETA(epoch): 306s + [Epoch 29][60/66] loss=0.938829 avg=0.863723 VRAM=38.8GiB | 6.0% done | ETA(epoch): 115s + Train loss: 0.861819 (1264.1s) ETA: 10896min + Val loss: 0.866379 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9876 t_0.4-0.6=0.8561 t_0.6-0.8=0.7586 t_0.8-1.0=0.7004] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 30/499 (6% done) --- + [Epoch 30][10/66] loss=0.780720 avg=0.891705 VRAM=39.0GiB | 6.0% done | ETA(epoch): 1069s + [Epoch 30][20/66] loss=0.834619 avg=0.857466 VRAM=38.8GiB | 6.1% done | ETA(epoch): 878s + [Epoch 30][30/66] loss=0.781763 avg=0.852829 VRAM=39.0GiB | 6.1% done | ETA(epoch): 687s + [Epoch 30][40/66] loss=0.886304 avg=0.852320 VRAM=38.8GiB | 6.1% done | ETA(epoch): 496s + [Epoch 30][50/66] loss=0.840431 avg=0.865754 VRAM=39.0GiB | 6.2% done | ETA(epoch): 305s + [Epoch 30][60/66] loss=0.791126 avg=0.859196 VRAM=38.8GiB | 6.2% done | ETA(epoch): 115s + Train loss: 0.861579 (1260.2s) ETA: 10864min + Val loss: 0.868022 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0108 t_0.4-0.6=0.8403 t_0.6-0.8=0.7269 t_0.8-1.0=0.6770] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 31/499 (6% done) --- + [Epoch 31][10/66] loss=0.822772 avg=0.861569 VRAM=39.0GiB | 6.2% done | ETA(epoch): 1069s + [Epoch 31][20/66] loss=0.742137 avg=0.850894 VRAM=38.8GiB | 6.3% done | ETA(epoch): 882s + [Epoch 31][30/66] loss=0.910846 avg=0.855610 VRAM=39.0GiB | 6.3% done | ETA(epoch): 689s + [Epoch 31][40/66] loss=0.919527 avg=0.863313 VRAM=38.8GiB | 6.3% done | ETA(epoch): 497s + [Epoch 31][50/66] loss=0.979510 avg=0.875586 VRAM=39.0GiB | 6.4% done | ETA(epoch): 306s + [Epoch 31][60/66] loss=0.802112 avg=0.873376 VRAM=38.8GiB | 6.4% done | ETA(epoch): 115s + Train loss: 0.875322 (1261.3s) ETA: 10832min + Val loss: 0.828917 [t_0.0-0.2=1.0808 t_0.2-0.4=1.0320 t_0.4-0.6=0.8275 t_0.6-0.8=0.7117 t_0.8-1.0=0.6927] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0031 (BEST) + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 32/499 (6% done) --- + [Epoch 32][10/66] loss=0.790474 avg=0.850034 VRAM=39.0GiB | 6.4% done | ETA(epoch): 1068s + [Epoch 32][20/66] loss=0.743303 avg=0.843878 VRAM=38.8GiB | 6.5% done | ETA(epoch): 882s + [Epoch 32][30/66] loss=0.989269 avg=0.860817 VRAM=39.0GiB | 6.5% done | ETA(epoch): 690s + [Epoch 32][40/66] loss=0.936098 avg=0.867722 VRAM=38.8GiB | 6.5% done | ETA(epoch): 498s + [Epoch 32][50/66] loss=0.737005 avg=0.860563 VRAM=39.0GiB | 6.6% done | ETA(epoch): 306s + [Epoch 32][60/66] loss=1.017199 avg=0.861351 VRAM=38.8GiB | 6.6% done | ETA(epoch): 115s + Train loss: 0.859653 (1261.9s) ETA: 10801min + Val loss: 0.880593 [t_0.0-0.2=1.0788 t_0.2-0.4=1.0331 t_0.4-0.6=0.8468 t_0.6-0.8=0.7324 t_0.8-1.0=0.6905] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 33/499 (7% done) --- + [Epoch 33][10/66] loss=0.760807 avg=0.867072 VRAM=39.0GiB | 6.6% done | ETA(epoch): 1069s + [Epoch 33][20/66] loss=0.956017 avg=0.842193 VRAM=38.8GiB | 6.7% done | ETA(epoch): 879s + [Epoch 33][30/66] loss=0.872623 avg=0.840261 VRAM=39.0GiB | 6.7% done | ETA(epoch): 688s + [Epoch 33][40/66] loss=0.846239 avg=0.840391 VRAM=38.8GiB | 6.7% done | ETA(epoch): 497s + [Epoch 33][50/66] loss=0.905481 avg=0.851699 VRAM=39.0GiB | 6.8% done | ETA(epoch): 306s + [Epoch 33][60/66] loss=0.764977 avg=0.849880 VRAM=38.8GiB | 6.8% done | ETA(epoch): 115s + Train loss: 0.851903 (1261.0s) ETA: 10771min + Val loss: 0.872860 [t_0.0-0.2=1.0868 t_0.2-0.4=1.0052 t_0.4-0.6=0.8499 t_0.6-0.8=0.7164 t_0.8-1.0=0.6707] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 34/499 (7% done) --- + [Epoch 34][10/66] loss=0.816713 avg=0.855252 VRAM=39.0GiB | 6.8% done | ETA(epoch): 1070s + [Epoch 34][20/66] loss=0.897059 avg=0.856750 VRAM=38.8GiB | 6.9% done | ETA(epoch): 879s + [Epoch 34][30/66] loss=0.855779 avg=0.843377 VRAM=39.0GiB | 6.9% done | ETA(epoch): 688s + [Epoch 34][40/66] loss=0.929487 avg=0.843081 VRAM=38.8GiB | 6.9% done | ETA(epoch): 497s + [Epoch 34][50/66] loss=0.786205 avg=0.855292 VRAM=39.0GiB | 7.0% done | ETA(epoch): 306s + [Epoch 34][60/66] loss=0.814949 avg=0.858973 VRAM=38.8GiB | 7.0% done | ETA(epoch): 115s + Train loss: 0.856656 (1261.4s) ETA: 10741min + Val loss: 0.855071 [t_0.0-0.2=1.0810 t_0.2-0.4=0.9810 t_0.4-0.6=0.8249 t_0.6-0.8=0.7299 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0034 +[MEM @ epoch 34 end] RAM: 17.4/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 35/499 (7% done) --- + [Epoch 35][10/66] loss=0.824940 avg=0.928500 VRAM=39.0GiB | 7.0% done | ETA(epoch): 1074s + [Epoch 35][20/66] loss=0.922620 avg=0.906516 VRAM=38.8GiB | 7.1% done | ETA(epoch): 884s + [Epoch 35][30/66] loss=0.904174 avg=0.893219 VRAM=39.0GiB | 7.1% done | ETA(epoch): 692s + [Epoch 35][40/66] loss=0.935529 avg=0.886677 VRAM=38.8GiB | 7.1% done | ETA(epoch): 499s + [Epoch 35][50/66] loss=0.817416 avg=0.874131 VRAM=39.0GiB | 7.2% done | ETA(epoch): 307s + [Epoch 35][60/66] loss=0.896516 avg=0.874374 VRAM=38.8GiB | 7.2% done | ETA(epoch): 115s + Train loss: 0.872228 (1265.9s) ETA: 10712min + Val loss: 0.851470 [t_0.0-0.2=1.0507 t_0.2-0.4=1.0064 t_0.4-0.6=0.8237 t_0.6-0.8=0.7382 t_0.8-1.0=0.6649] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 17.2/188.5 GiB (9.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 36/499 (7% done) --- + [Epoch 36][10/66] loss=0.763807 avg=0.853137 VRAM=39.0GiB | 7.2% done | ETA(epoch): 1070s + [Epoch 36][20/66] loss=0.934546 avg=0.876624 VRAM=38.8GiB | 7.3% done | ETA(epoch): 879s + [Epoch 36][30/66] loss=0.917519 avg=0.872169 VRAM=39.0GiB | 7.3% done | ETA(epoch): 688s + [Epoch 36][40/66] loss=0.971477 avg=0.872672 VRAM=38.8GiB | 7.3% done | ETA(epoch): 497s + [Epoch 36][50/66] loss=0.946561 avg=0.871503 VRAM=39.0GiB | 7.4% done | ETA(epoch): 306s + [Epoch 36][60/66] loss=0.827615 avg=0.869501 VRAM=38.8GiB | 7.4% done | ETA(epoch): 115s + Train loss: 0.868861 (1260.4s) ETA: 10683min + Val loss: 0.851203 [t_0.0-0.2=1.0714 t_0.2-0.4=1.0291 t_0.4-0.6=0.8855 t_0.6-0.8=0.7056 t_0.8-1.0=0.6670] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 37/499 (7% done) --- + [Epoch 37][10/66] loss=0.846416 avg=0.874083 VRAM=39.0GiB | 7.4% done | ETA(epoch): 1068s + [Epoch 37][20/66] loss=0.913408 avg=0.891302 VRAM=38.8GiB | 7.5% done | ETA(epoch): 877s + [Epoch 37][30/66] loss=0.925050 avg=0.893465 VRAM=39.0GiB | 7.5% done | ETA(epoch): 687s + [Epoch 37][40/66] loss=0.924096 avg=0.879735 VRAM=38.8GiB | 7.5% done | ETA(epoch): 508s + [Epoch 37][50/66] loss=0.915857 avg=0.872851 VRAM=39.0GiB | 7.6% done | ETA(epoch): 311s + [Epoch 37][60/66] loss=0.941961 avg=0.875241 VRAM=38.8GiB | 7.6% done | ETA(epoch): 116s + Train loss: 0.873222 (1277.6s) ETA: 10657min + Val loss: 0.863573 [t_0.0-0.2=1.0801 t_0.2-0.4=0.9743 t_0.4-0.6=0.8578 t_0.6-0.8=0.7312 t_0.8-1.0=0.6896] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 38/499 (8% done) --- + [Epoch 38][10/66] loss=0.819804 avg=0.874146 VRAM=39.0GiB | 7.6% done | ETA(epoch): 1069s + [Epoch 38][20/66] loss=0.780206 avg=0.853159 VRAM=38.8GiB | 7.7% done | ETA(epoch): 878s + [Epoch 38][30/66] loss=0.960263 avg=0.861826 VRAM=39.0GiB | 7.7% done | ETA(epoch): 690s + [Epoch 38][40/66] loss=0.952048 avg=0.864547 VRAM=38.8GiB | 7.7% done | ETA(epoch): 498s + [Epoch 38][50/66] loss=1.041423 avg=0.867806 VRAM=39.0GiB | 7.8% done | ETA(epoch): 306s + [Epoch 38][60/66] loss=0.959938 avg=0.866882 VRAM=38.8GiB | 7.8% done | ETA(epoch): 115s + Train loss: 0.862701 (1262.3s) ETA: 10629min + Val loss: 0.852566 [t_0.0-0.2=1.0934 t_0.2-0.4=0.9929 t_0.4-0.6=0.8438 t_0.6-0.8=0.7289 t_0.8-1.0=0.6777] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 39/499 (8% done) --- + [Epoch 39][10/66] loss=0.915149 avg=0.877545 VRAM=39.0GiB | 7.8% done | ETA(epoch): 1068s + [Epoch 39][20/66] loss=0.734530 avg=0.874929 VRAM=38.8GiB | 7.9% done | ETA(epoch): 885s + [Epoch 39][30/66] loss=0.997476 avg=0.869522 VRAM=39.0GiB | 7.9% done | ETA(epoch): 691s + [Epoch 39][40/66] loss=0.839733 avg=0.864958 VRAM=38.8GiB | 7.9% done | ETA(epoch): 498s + [Epoch 39][50/66] loss=0.961883 avg=0.867145 VRAM=39.0GiB | 8.0% done | ETA(epoch): 306s + [Epoch 39][60/66] loss=0.924932 avg=0.867007 VRAM=38.8GiB | 8.0% done | ETA(epoch): 115s + Train loss: 0.864124 (1262.4s) ETA: 10601min + Val loss: 0.861838 [t_0.0-0.2=1.0887 t_0.2-0.4=0.9749 t_0.4-0.6=0.8661 t_0.6-0.8=0.7019 t_0.8-1.0=0.6696] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 40/499 (8% done) --- + [Epoch 40][10/66] loss=0.895005 avg=0.876613 VRAM=39.0GiB | 8.0% done | ETA(epoch): 1068s + [Epoch 40][20/66] loss=0.823711 avg=0.868805 VRAM=38.8GiB | 8.1% done | ETA(epoch): 877s + [Epoch 40][30/66] loss=0.838376 avg=0.870810 VRAM=39.0GiB | 8.1% done | ETA(epoch): 687s + [Epoch 40][40/66] loss=1.012366 avg=0.870252 VRAM=38.8GiB | 8.1% done | ETA(epoch): 496s + [Epoch 40][50/66] loss=0.977117 avg=0.875349 VRAM=39.0GiB | 8.2% done | ETA(epoch): 305s + [Epoch 40][60/66] loss=0.847001 avg=0.871289 VRAM=38.8GiB | 8.2% done | ETA(epoch): 115s + Train loss: 0.867548 (1259.7s) ETA: 10572min + Val loss: 0.897370 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9794 t_0.4-0.6=0.8515 t_0.6-0.8=0.7252 t_0.8-1.0=0.6992] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 41/499 (8% done) --- + [Epoch 41][10/66] loss=0.992604 avg=0.875803 VRAM=39.0GiB | 8.2% done | ETA(epoch): 1069s + [Epoch 41][20/66] loss=0.980769 avg=0.896247 VRAM=38.8GiB | 8.3% done | ETA(epoch): 878s + [Epoch 41][30/66] loss=0.800908 avg=0.888875 VRAM=39.0GiB | 8.3% done | ETA(epoch): 687s + [Epoch 41][40/66] loss=0.832435 avg=0.879680 VRAM=38.8GiB | 8.3% done | ETA(epoch): 496s + [Epoch 41][50/66] loss=0.830968 avg=0.875609 VRAM=39.0GiB | 8.4% done | ETA(epoch): 305s + [Epoch 41][60/66] loss=0.814581 avg=0.877623 VRAM=38.8GiB | 8.4% done | ETA(epoch): 115s + Train loss: 0.879669 (1260.2s) ETA: 10544min + Val loss: 0.847661 [t_0.0-0.2=1.0654 t_0.2-0.4=0.9904 t_0.4-0.6=0.8304 t_0.6-0.8=0.7346 t_0.8-1.0=0.6885] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 42/499 (8% done) --- + [Epoch 42][10/66] loss=0.789884 avg=0.828809 VRAM=39.0GiB | 8.4% done | ETA(epoch): 1070s + [Epoch 42][20/66] loss=0.948202 avg=0.843425 VRAM=38.8GiB | 8.5% done | ETA(epoch): 879s + [Epoch 42][30/66] loss=0.868117 avg=0.846614 VRAM=39.0GiB | 8.5% done | ETA(epoch): 688s + [Epoch 42][40/66] loss=0.820427 avg=0.844120 VRAM=38.8GiB | 8.5% done | ETA(epoch): 497s + [Epoch 42][50/66] loss=0.750398 avg=0.842043 VRAM=39.0GiB | 8.6% done | ETA(epoch): 306s + [Epoch 42][60/66] loss=0.859535 avg=0.846981 VRAM=38.8GiB | 8.6% done | ETA(epoch): 115s + Train loss: 0.849393 (1263.1s) ETA: 10517min + Val loss: 0.861461 [t_0.0-0.2=1.0830 t_0.2-0.4=0.9613 t_0.4-0.6=0.8720 t_0.6-0.8=0.7256 t_0.8-1.0=0.6837] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 43/499 (9% done) --- + [Epoch 43][10/66] loss=0.862585 avg=0.809040 VRAM=39.0GiB | 8.6% done | ETA(epoch): 1069s + [Epoch 43][20/66] loss=0.796383 avg=0.841037 VRAM=38.8GiB | 8.7% done | ETA(epoch): 878s + [Epoch 43][30/66] loss=0.814297 avg=0.844060 VRAM=39.0GiB | 8.7% done | ETA(epoch): 687s + [Epoch 43][40/66] loss=0.783859 avg=0.847849 VRAM=38.8GiB | 8.7% done | ETA(epoch): 496s + [Epoch 43][50/66] loss=0.956222 avg=0.852116 VRAM=39.0GiB | 8.8% done | ETA(epoch): 306s + [Epoch 43][60/66] loss=0.806879 avg=0.854214 VRAM=38.8GiB | 8.8% done | ETA(epoch): 115s + Train loss: 0.853001 (1260.2s) ETA: 10489min + Val loss: 0.850108 [t_0.0-0.2=1.0843 t_0.2-0.4=1.0041 t_0.4-0.6=0.8364 t_0.6-0.8=0.7349 t_0.8-1.0=0.6656] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 44/499 (9% done) --- + [Epoch 44][10/66] loss=0.889040 avg=0.840406 VRAM=39.0GiB | 8.8% done | ETA(epoch): 1068s + [Epoch 44][20/66] loss=0.865110 avg=0.841975 VRAM=38.8GiB | 8.9% done | ETA(epoch): 879s + [Epoch 44][30/66] loss=0.739255 avg=0.839724 VRAM=39.0GiB | 8.9% done | ETA(epoch): 688s + [Epoch 44][40/66] loss=0.736071 avg=0.833992 VRAM=38.8GiB | 8.9% done | ETA(epoch): 497s + [Epoch 44][50/66] loss=0.864688 avg=0.838172 VRAM=39.0GiB | 9.0% done | ETA(epoch): 306s + [Epoch 44][60/66] loss=0.998152 avg=0.847997 VRAM=38.8GiB | 9.0% done | ETA(epoch): 115s + Train loss: 0.850607 (1260.5s) ETA: 10462min + Val loss: 0.848041 [t_0.0-0.2=1.0648 t_0.2-0.4=1.0423 t_0.4-0.6=0.8679 t_0.6-0.8=0.7210 t_0.8-1.0=0.6542] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 45/499 (9% done) --- + [Epoch 45][10/66] loss=0.801797 avg=0.889522 VRAM=39.0GiB | 9.0% done | ETA(epoch): 1068s + [Epoch 45][20/66] loss=0.798412 avg=0.877350 VRAM=38.8GiB | 9.1% done | ETA(epoch): 878s + [Epoch 45][30/66] loss=0.836200 avg=0.868516 VRAM=39.0GiB | 9.1% done | ETA(epoch): 687s + [Epoch 45][40/66] loss=0.957268 avg=0.873619 VRAM=38.8GiB | 9.1% done | ETA(epoch): 498s + [Epoch 45][50/66] loss=0.804535 avg=0.874399 VRAM=39.0GiB | 9.2% done | ETA(epoch): 306s + [Epoch 45][60/66] loss=0.939651 avg=0.872886 VRAM=38.8GiB | 9.2% done | ETA(epoch): 115s + Train loss: 0.870925 (1263.6s) ETA: 10436min + Val loss: 0.827523 [t_0.0-0.2=1.0488 t_0.2-0.4=1.0211 t_0.4-0.6=0.8698 t_0.6-0.8=0.7088 t_0.8-1.0=0.6645] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0045 (BEST) + Deleted old checkpoint: checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 46/499 (9% done) --- + [Epoch 46][10/66] loss=0.931708 avg=0.872054 VRAM=39.0GiB | 9.2% done | ETA(epoch): 1073s + [Epoch 46][20/66] loss=0.862294 avg=0.857503 VRAM=38.8GiB | 9.3% done | ETA(epoch): 881s + [Epoch 46][30/66] loss=0.784662 avg=0.859292 VRAM=39.0GiB | 9.3% done | ETA(epoch): 692s + [Epoch 46][40/66] loss=0.881821 avg=0.856143 VRAM=38.8GiB | 9.3% done | ETA(epoch): 499s + [Epoch 46][50/66] loss=0.798922 avg=0.855479 VRAM=39.0GiB | 9.4% done | ETA(epoch): 307s + [Epoch 46][60/66] loss=0.887376 avg=0.857706 VRAM=38.8GiB | 9.4% done | ETA(epoch): 115s + Train loss: 0.854246 (1267.1s) ETA: 10410min + Val loss: 0.857945 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0178 t_0.4-0.6=0.8740 t_0.6-0.8=0.7268 t_0.8-1.0=0.6817] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 47/499 (9% done) --- + [Epoch 47][10/66] loss=0.873295 avg=0.870595 VRAM=39.0GiB | 9.4% done | ETA(epoch): 1073s + [Epoch 47][20/66] loss=0.706372 avg=0.848916 VRAM=38.8GiB | 9.5% done | ETA(epoch): 886s + [Epoch 47][30/66] loss=0.886073 avg=0.853677 VRAM=39.0GiB | 9.5% done | ETA(epoch): 692s + [Epoch 47][40/66] loss=0.970366 avg=0.855452 VRAM=38.8GiB | 9.5% done | ETA(epoch): 499s + [Epoch 47][50/66] loss=0.824682 avg=0.859154 VRAM=39.0GiB | 9.6% done | ETA(epoch): 307s + [Epoch 47][60/66] loss=0.960196 avg=0.862168 VRAM=38.8GiB | 9.6% done | ETA(epoch): 115s + Train loss: 0.862997 (1266.3s) ETA: 10384min + Val loss: 0.853540 [t_0.0-0.2=1.0949 t_0.2-0.4=0.9872 t_0.4-0.6=0.8479 t_0.6-0.8=0.7054 t_0.8-1.0=0.6808] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 48/499 (10% done) --- + [Epoch 48][10/66] loss=1.009247 avg=0.864932 VRAM=39.0GiB | 9.6% done | ETA(epoch): 1073s + [Epoch 48][20/66] loss=0.798284 avg=0.868536 VRAM=38.8GiB | 9.7% done | ETA(epoch): 882s + [Epoch 48][30/66] loss=0.980788 avg=0.868087 VRAM=39.0GiB | 9.7% done | ETA(epoch): 690s + [Epoch 48][40/66] loss=0.985038 avg=0.872365 VRAM=38.8GiB | 9.7% done | ETA(epoch): 498s + [Epoch 48][50/66] loss=1.011543 avg=0.876512 VRAM=39.0GiB | 9.8% done | ETA(epoch): 307s + [Epoch 48][60/66] loss=0.915721 avg=0.878353 VRAM=38.8GiB | 9.8% done | ETA(epoch): 115s + Train loss: 0.875998 (1264.8s) ETA: 10358min + Val loss: 0.843156 [t_0.0-0.2=1.0652 t_0.2-0.4=1.0009 t_0.4-0.6=0.8406 t_0.6-0.8=0.7160 t_0.8-1.0=0.6726] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0048 +[MEM @ epoch 48 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 49/499 (10% done) --- + [Epoch 49][10/66] loss=0.870314 avg=0.894138 VRAM=39.0GiB | 9.8% done | ETA(epoch): 1090s + [Epoch 49][20/66] loss=0.792892 avg=0.880764 VRAM=38.8GiB | 9.9% done | ETA(epoch): 890s + [Epoch 49][30/66] loss=0.802301 avg=0.861920 VRAM=39.0GiB | 9.9% done | ETA(epoch): 696s + [Epoch 49][40/66] loss=0.865949 avg=0.864116 VRAM=38.8GiB | 9.9% done | ETA(epoch): 502s + [Epoch 49][50/66] loss=0.866877 avg=0.873378 VRAM=39.0GiB | 10.0% done | ETA(epoch): 309s + [Epoch 49][60/66] loss=0.972561 avg=0.880614 VRAM=38.8GiB | 10.0% done | ETA(epoch): 116s + Train loss: 0.879057 (1272.7s) ETA: 10334min + Val loss: 0.848242 [t_0.0-0.2=1.0651 t_0.2-0.4=1.0044 t_0.4-0.6=0.8500 t_0.6-0.8=0.7022 t_0.8-1.0=0.6760] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 50/499 (10% done) --- + [MilestoneVis] train_0 step 3300 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3300 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3300 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3300 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3300 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3300 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3300 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3300 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3300 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3300 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 50][10/66] loss=0.719460 avg=0.839227 VRAM=39.0GiB | 10.0% done | ETA(epoch): 6804s + [Epoch 50][20/66] loss=0.786088 avg=0.846602 VRAM=38.8GiB | 10.1% done | ETA(epoch): 3236s + [Epoch 50][30/66] loss=0.768648 avg=0.849443 VRAM=39.0GiB | 10.1% done | ETA(epoch): 1919s + [Epoch 50][40/66] loss=0.945295 avg=0.856880 VRAM=38.8GiB | 10.1% done | ETA(epoch): 1164s + [Epoch 50][50/66] loss=0.842516 avg=0.857697 VRAM=39.0GiB | 10.2% done | ETA(epoch): 634s + [Epoch 50][60/66] loss=0.733345 avg=0.854995 VRAM=38.8GiB | 10.2% done | ETA(epoch): 217s + Train loss: 0.860127 (2292.7s) ETA: 10459min + Val loss: 0.830167 [t_0.0-0.2=1.0645 t_0.2-0.4=0.9713 t_0.4-0.6=0.8511 t_0.6-0.8=0.7244 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 51/499 (10% done) --- + [Epoch 51][10/66] loss=0.875851 avg=0.870883 VRAM=39.0GiB | 10.2% done | ETA(epoch): 1073s + [Epoch 51][20/66] loss=0.841827 avg=0.880273 VRAM=38.8GiB | 10.3% done | ETA(epoch): 882s + [Epoch 51][30/66] loss=0.816870 avg=0.857278 VRAM=39.0GiB | 10.3% done | ETA(epoch): 690s + [Epoch 51][40/66] loss=0.955567 avg=0.855895 VRAM=38.8GiB | 10.3% done | ETA(epoch): 498s + [Epoch 51][50/66] loss=0.867074 avg=0.863436 VRAM=39.0GiB | 10.4% done | ETA(epoch): 307s + [Epoch 51][60/66] loss=0.979836 avg=0.866735 VRAM=38.8GiB | 10.4% done | ETA(epoch): 115s + Train loss: 0.869016 (1264.4s) ETA: 10430min + Val loss: 0.844844 [t_0.0-0.2=1.0724 t_0.2-0.4=1.0012 t_0.4-0.6=0.8136 t_0.6-0.8=0.7401 t_0.8-1.0=0.6490] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 52/499 (10% done) --- + [Epoch 52][10/66] loss=0.889591 avg=0.860935 VRAM=39.0GiB | 10.4% done | ETA(epoch): 1073s + [Epoch 52][20/66] loss=0.828666 avg=0.844988 VRAM=38.8GiB | 10.5% done | ETA(epoch): 882s + [Epoch 52][30/66] loss=0.853324 avg=0.834845 VRAM=39.0GiB | 10.5% done | ETA(epoch): 690s + [Epoch 52][40/66] loss=0.885725 avg=0.843946 VRAM=38.8GiB | 10.5% done | ETA(epoch): 498s + [Epoch 52][50/66] loss=0.694576 avg=0.839410 VRAM=39.0GiB | 10.6% done | ETA(epoch): 307s + [Epoch 52][60/66] loss=0.925247 avg=0.844714 VRAM=38.8GiB | 10.6% done | ETA(epoch): 115s + Train loss: 0.846745 (1265.0s) ETA: 10401min + Val loss: 0.884265 [t_0.0-0.2=1.0665 t_0.2-0.4=0.9830 t_0.4-0.6=0.8232 t_0.6-0.8=0.7204 t_0.8-1.0=0.6730] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 53/499 (11% done) --- + [Epoch 53][10/66] loss=0.899684 avg=0.859153 VRAM=39.0GiB | 10.6% done | ETA(epoch): 1072s + [Epoch 53][20/66] loss=0.815470 avg=0.868640 VRAM=38.8GiB | 10.7% done | ETA(epoch): 882s + [Epoch 53][30/66] loss=0.821530 avg=0.859317 VRAM=39.0GiB | 10.7% done | ETA(epoch): 690s + [Epoch 53][40/66] loss=0.945209 avg=0.858689 VRAM=38.8GiB | 10.7% done | ETA(epoch): 498s + [Epoch 53][50/66] loss=0.796167 avg=0.854061 VRAM=39.0GiB | 10.8% done | ETA(epoch): 307s + [Epoch 53][60/66] loss=0.762038 avg=0.843834 VRAM=38.8GiB | 10.8% done | ETA(epoch): 115s + Train loss: 0.844149 (1265.1s) ETA: 10373min + Val loss: 0.842175 [t_0.0-0.2=1.0786 t_0.2-0.4=0.9542 t_0.4-0.6=0.8375 t_0.6-0.8=0.7446 t_0.8-1.0=0.6566] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 54/499 (11% done) --- + [Epoch 54][10/66] loss=0.786594 avg=0.841631 VRAM=39.0GiB | 10.8% done | ETA(epoch): 1073s + [Epoch 54][20/66] loss=0.811349 avg=0.854479 VRAM=38.8GiB | 10.9% done | ETA(epoch): 881s + [Epoch 54][30/66] loss=0.878536 avg=0.854594 VRAM=39.0GiB | 10.9% done | ETA(epoch): 690s + [Epoch 54][40/66] loss=0.883894 avg=0.850882 VRAM=38.8GiB | 10.9% done | ETA(epoch): 498s + [Epoch 54][50/66] loss=0.931979 avg=0.856203 VRAM=39.0GiB | 11.0% done | ETA(epoch): 307s + [Epoch 54][60/66] loss=0.736154 avg=0.856741 VRAM=38.8GiB | 11.0% done | ETA(epoch): 115s + Train loss: 0.854601 (1265.2s) ETA: 10345min + Val loss: 0.868478 [t_0.0-0.2=1.0572 t_0.2-0.4=1.0143 t_0.4-0.6=0.8652 t_0.6-0.8=0.7289 t_0.8-1.0=0.6629] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 55/499 (11% done) --- + [Epoch 55][10/66] loss=0.880616 avg=0.891214 VRAM=39.0GiB | 11.0% done | ETA(epoch): 1071s + [Epoch 55][20/66] loss=0.820032 avg=0.874886 VRAM=38.8GiB | 11.1% done | ETA(epoch): 880s + [Epoch 55][30/66] loss=0.958907 avg=0.873379 VRAM=39.0GiB | 11.1% done | ETA(epoch): 692s + [Epoch 55][40/66] loss=0.936940 avg=0.870016 VRAM=38.8GiB | 11.1% done | ETA(epoch): 499s + [Epoch 55][50/66] loss=0.929907 avg=0.872937 VRAM=39.0GiB | 11.2% done | ETA(epoch): 307s + [Epoch 55][60/66] loss=0.822202 avg=0.870660 VRAM=38.8GiB | 11.2% done | ETA(epoch): 115s + Train loss: 0.869721 (1266.7s) ETA: 10318min + Val loss: 0.871528 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0220 t_0.4-0.6=0.8503 t_0.6-0.8=0.7175 t_0.8-1.0=0.6628] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 56/499 (11% done) --- + [Epoch 56][10/66] loss=0.963243 avg=0.869769 VRAM=39.0GiB | 11.2% done | ETA(epoch): 1073s + [Epoch 56][20/66] loss=0.951036 avg=0.860447 VRAM=38.8GiB | 11.3% done | ETA(epoch): 882s + [Epoch 56][30/66] loss=0.791976 avg=0.849303 VRAM=39.0GiB | 11.3% done | ETA(epoch): 690s + [Epoch 56][40/66] loss=0.832278 avg=0.865942 VRAM=38.8GiB | 11.3% done | ETA(epoch): 499s + [Epoch 56][50/66] loss=0.715159 avg=0.863993 VRAM=39.0GiB | 11.4% done | ETA(epoch): 307s + [Epoch 56][60/66] loss=0.924183 avg=0.865031 VRAM=38.8GiB | 11.4% done | ETA(epoch): 115s + Train loss: 0.865467 (1266.5s) ETA: 10290min + Val loss: 0.855192 [t_0.0-0.2=1.0553 t_0.2-0.4=1.0050 t_0.4-0.6=0.8322 t_0.6-0.8=0.7362 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 57/499 (11% done) --- + [Epoch 57][10/66] loss=0.786937 avg=0.825446 VRAM=39.0GiB | 11.4% done | ETA(epoch): 1073s + [Epoch 57][20/66] loss=0.988155 avg=0.831816 VRAM=38.8GiB | 11.5% done | ETA(epoch): 882s + [Epoch 57][30/66] loss=0.703228 avg=0.843792 VRAM=39.0GiB | 11.5% done | ETA(epoch): 690s + [Epoch 57][40/66] loss=0.777449 avg=0.841134 VRAM=38.8GiB | 11.5% done | ETA(epoch): 498s + [Epoch 57][50/66] loss=0.744213 avg=0.840558 VRAM=39.0GiB | 11.6% done | ETA(epoch): 307s + [Epoch 57][60/66] loss=0.765377 avg=0.844689 VRAM=38.8GiB | 11.6% done | ETA(epoch): 115s + Train loss: 0.843826 (1265.6s) ETA: 10263min + Val loss: 0.860142 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9888 t_0.4-0.6=0.8450 t_0.6-0.8=0.7319 t_0.8-1.0=0.6531] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 16.5/188.5 GiB (8.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 58/499 (12% done) --- + [Epoch 58][10/66] loss=0.802907 avg=0.865738 VRAM=39.0GiB | 11.6% done | ETA(epoch): 1072s + [Epoch 58][20/66] loss=0.962486 avg=0.867826 VRAM=38.8GiB | 11.7% done | ETA(epoch): 882s + [Epoch 58][30/66] loss=0.913371 avg=0.867992 VRAM=39.0GiB | 11.7% done | ETA(epoch): 691s + [Epoch 58][40/66] loss=0.793218 avg=0.865610 VRAM=38.8GiB | 11.7% done | ETA(epoch): 499s + [Epoch 58][50/66] loss=1.000310 avg=0.869878 VRAM=39.0GiB | 11.8% done | ETA(epoch): 307s + [Epoch 58][60/66] loss=0.804084 avg=0.872696 VRAM=38.8GiB | 11.8% done | ETA(epoch): 115s + Train loss: 0.871332 (1269.6s) ETA: 10236min + Val loss: 0.839203 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9977 t_0.4-0.6=0.8691 t_0.6-0.8=0.7113 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 59/499 (12% done) --- + [Epoch 59][10/66] loss=0.729358 avg=0.875188 VRAM=39.0GiB | 11.8% done | ETA(epoch): 1074s + [Epoch 59][20/66] loss=0.781316 avg=0.861690 VRAM=38.8GiB | 11.9% done | ETA(epoch): 883s + [Epoch 59][30/66] loss=0.928625 avg=0.858783 VRAM=39.0GiB | 11.9% done | ETA(epoch): 691s + [Epoch 59][40/66] loss=0.762885 avg=0.858752 VRAM=38.8GiB | 11.9% done | ETA(epoch): 499s + [Epoch 59][50/66] loss=0.941751 avg=0.858917 VRAM=39.0GiB | 12.0% done | ETA(epoch): 307s + [Epoch 59][60/66] loss=0.911087 avg=0.860095 VRAM=38.8GiB | 12.0% done | ETA(epoch): 115s + Train loss: 0.851977 (1267.0s) ETA: 10209min + Val loss: 0.886479 [t_0.0-0.2=1.0664 t_0.2-0.4=0.9717 t_0.4-0.6=0.8662 t_0.6-0.8=0.7360 t_0.8-1.0=0.6528] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 60/499 (12% done) --- + [Epoch 60][10/66] loss=0.862218 avg=0.865247 VRAM=39.0GiB | 12.0% done | ETA(epoch): 1073s + [Epoch 60][20/66] loss=0.904572 avg=0.861214 VRAM=38.8GiB | 12.1% done | ETA(epoch): 882s + [Epoch 60][30/66] loss=0.813681 avg=0.871100 VRAM=39.0GiB | 12.1% done | ETA(epoch): 690s + [Epoch 60][40/66] loss=0.927090 avg=0.857844 VRAM=38.8GiB | 12.1% done | ETA(epoch): 498s + [Epoch 60][50/66] loss=0.733275 avg=0.854498 VRAM=39.0GiB | 12.2% done | ETA(epoch): 307s + [Epoch 60][60/66] loss=0.851814 avg=0.855008 VRAM=38.8GiB | 12.2% done | ETA(epoch): 115s + Train loss: 0.860034 (1267.4s) ETA: 10182min + Val loss: 0.841326 [t_0.0-0.2=1.0818 t_0.2-0.4=1.0043 t_0.4-0.6=0.8618 t_0.6-0.8=0.7055 t_0.8-1.0=0.6586] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 61/499 (12% done) --- + [Epoch 61][10/66] loss=0.839671 avg=0.837608 VRAM=39.0GiB | 12.2% done | ETA(epoch): 1073s + [Epoch 61][20/66] loss=0.762953 avg=0.850566 VRAM=38.8GiB | 12.3% done | ETA(epoch): 881s + [Epoch 61][30/66] loss=0.934654 avg=0.863881 VRAM=39.0GiB | 12.3% done | ETA(epoch): 690s + [Epoch 61][40/66] loss=0.881470 avg=0.858198 VRAM=38.8GiB | 12.3% done | ETA(epoch): 498s + [Epoch 61][50/66] loss=0.692309 avg=0.856430 VRAM=39.0GiB | 12.4% done | ETA(epoch): 307s + [Epoch 61][60/66] loss=0.918994 avg=0.858387 VRAM=38.8GiB | 12.4% done | ETA(epoch): 115s + Train loss: 0.858654 (1264.9s) ETA: 10155min + Val loss: 0.854329 [t_0.0-0.2=1.0784 t_0.2-0.4=0.9994 t_0.4-0.6=0.8656 t_0.6-0.8=0.7242 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 62/499 (12% done) --- + [Epoch 62][10/66] loss=0.879156 avg=0.875958 VRAM=39.0GiB | 12.4% done | ETA(epoch): 1074s + [Epoch 62][20/66] loss=0.746356 avg=0.862123 VRAM=38.8GiB | 12.5% done | ETA(epoch): 882s + [Epoch 62][30/66] loss=0.902347 avg=0.857633 VRAM=39.0GiB | 12.5% done | ETA(epoch): 690s + [Epoch 62][40/66] loss=0.798996 avg=0.853282 VRAM=38.8GiB | 12.5% done | ETA(epoch): 498s + [Epoch 62][50/66] loss=0.889570 avg=0.857325 VRAM=39.0GiB | 12.6% done | ETA(epoch): 307s + [Epoch 62][60/66] loss=0.769012 avg=0.862045 VRAM=38.8GiB | 12.6% done | ETA(epoch): 115s + Train loss: 0.863355 (1265.5s) ETA: 10128min + Val loss: 0.842665 [t_0.0-0.2=1.0806 t_0.2-0.4=1.0163 t_0.4-0.6=0.8543 t_0.6-0.8=0.7089 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 63/499 (13% done) --- + [Epoch 63][10/66] loss=0.796473 avg=0.845995 VRAM=39.0GiB | 12.6% done | ETA(epoch): 1073s + [Epoch 63][20/66] loss=0.821781 avg=0.840455 VRAM=38.8GiB | 12.7% done | ETA(epoch): 882s + [Epoch 63][30/66] loss=0.787096 avg=0.836198 VRAM=39.0GiB | 12.7% done | ETA(epoch): 690s + [Epoch 63][40/66] loss=0.929704 avg=0.845983 VRAM=38.8GiB | 12.7% done | ETA(epoch): 498s + [Epoch 63][50/66] loss=0.807839 avg=0.848710 VRAM=39.0GiB | 12.8% done | ETA(epoch): 307s + [Epoch 63][60/66] loss=0.693517 avg=0.852430 VRAM=38.8GiB | 12.8% done | ETA(epoch): 115s + Train loss: 0.852268 (1265.2s) ETA: 10101min + Val loss: 0.844960 [t_0.0-0.2=1.0526 t_0.2-0.4=0.9485 t_0.4-0.6=0.8405 t_0.6-0.8=0.7006 t_0.8-1.0=0.6484] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 64/499 (13% done) --- + [Epoch 64][10/66] loss=0.967195 avg=0.868691 VRAM=39.0GiB | 12.8% done | ETA(epoch): 1073s + [Epoch 64][20/66] loss=0.967164 avg=0.878165 VRAM=38.8GiB | 12.9% done | ETA(epoch): 888s + [Epoch 64][30/66] loss=0.794339 avg=0.868253 VRAM=39.0GiB | 12.9% done | ETA(epoch): 693s + [Epoch 64][40/66] loss=0.934039 avg=0.866204 VRAM=38.8GiB | 12.9% done | ETA(epoch): 500s + [Epoch 64][50/66] loss=0.833401 avg=0.860169 VRAM=39.0GiB | 13.0% done | ETA(epoch): 308s + [Epoch 64][60/66] loss=0.767562 avg=0.860805 VRAM=38.8GiB | 13.0% done | ETA(epoch): 115s + Train loss: 0.862297 (1267.6s) ETA: 10075min + Val loss: 0.838027 [t_0.0-0.2=1.0645 t_0.2-0.4=0.9576 t_0.4-0.6=0.8574 t_0.6-0.8=0.7127 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 65/499 (13% done) --- + [Epoch 65][10/66] loss=0.949755 avg=0.869896 VRAM=39.0GiB | 13.0% done | ETA(epoch): 1073s + [Epoch 65][20/66] loss=0.787969 avg=0.874071 VRAM=38.8GiB | 13.1% done | ETA(epoch): 882s + [Epoch 65][30/66] loss=0.805536 avg=0.867647 VRAM=39.0GiB | 13.1% done | ETA(epoch): 690s + [Epoch 65][40/66] loss=0.929070 avg=0.862006 VRAM=38.8GiB | 13.1% done | ETA(epoch): 498s + [Epoch 65][50/66] loss=0.795740 avg=0.858289 VRAM=39.0GiB | 13.2% done | ETA(epoch): 307s + [Epoch 65][60/66] loss=1.005185 avg=0.860296 VRAM=38.8GiB | 13.2% done | ETA(epoch): 115s + Train loss: 0.861890 (1264.6s) ETA: 10049min + Val loss: 0.868250 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9791 t_0.4-0.6=0.8963 t_0.6-0.8=0.7303 t_0.8-1.0=0.6635] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 66/499 (13% done) --- + [Epoch 66][10/66] loss=0.970906 avg=0.822478 VRAM=39.0GiB | 13.2% done | ETA(epoch): 1074s + [Epoch 66][20/66] loss=0.894948 avg=0.842984 VRAM=38.8GiB | 13.3% done | ETA(epoch): 882s + [Epoch 66][30/66] loss=0.876106 avg=0.841951 VRAM=39.0GiB | 13.3% done | ETA(epoch): 690s + [Epoch 66][40/66] loss=0.976342 avg=0.845956 VRAM=38.8GiB | 13.3% done | ETA(epoch): 498s + [Epoch 66][50/66] loss=0.863103 avg=0.851052 VRAM=39.0GiB | 13.4% done | ETA(epoch): 307s + [Epoch 66][60/66] loss=0.991141 avg=0.852481 VRAM=38.8GiB | 13.4% done | ETA(epoch): 115s + Train loss: 0.848161 (1267.2s) ETA: 10023min + Val loss: 0.859925 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9914 t_0.4-0.6=0.8668 t_0.6-0.8=0.7252 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 67/499 (13% done) --- + [Epoch 67][10/66] loss=0.895148 avg=0.876773 VRAM=39.0GiB | 13.4% done | ETA(epoch): 1073s + [Epoch 67][20/66] loss=0.898558 avg=0.851227 VRAM=38.8GiB | 13.5% done | ETA(epoch): 881s + [Epoch 67][30/66] loss=0.871623 avg=0.847696 VRAM=39.0GiB | 13.5% done | ETA(epoch): 690s + [Epoch 67][40/66] loss=0.804163 avg=0.839725 VRAM=38.8GiB | 13.5% done | ETA(epoch): 499s + [Epoch 67][50/66] loss=0.805720 avg=0.841079 VRAM=39.0GiB | 13.6% done | ETA(epoch): 307s + [Epoch 67][60/66] loss=0.902335 avg=0.844610 VRAM=38.8GiB | 13.6% done | ETA(epoch): 115s + Train loss: 0.845832 (1267.9s) ETA: 9997min + Val loss: 0.863283 [t_0.0-0.2=1.0752 t_0.2-0.4=0.9508 t_0.4-0.6=0.8491 t_0.6-0.8=0.7395 t_0.8-1.0=0.6633] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 68/499 (14% done) --- + [Epoch 68][10/66] loss=0.845003 avg=0.880513 VRAM=39.0GiB | 13.6% done | ETA(epoch): 1073s + [Epoch 68][20/66] loss=0.906444 avg=0.878902 VRAM=38.8GiB | 13.7% done | ETA(epoch): 882s + [Epoch 68][30/66] loss=0.886116 avg=0.869232 VRAM=39.0GiB | 13.7% done | ETA(epoch): 690s + [Epoch 68][40/66] loss=0.771948 avg=0.867655 VRAM=38.8GiB | 13.7% done | ETA(epoch): 499s + [Epoch 68][50/66] loss=0.752635 avg=0.865125 VRAM=39.0GiB | 13.8% done | ETA(epoch): 307s + [Epoch 68][60/66] loss=0.905638 avg=0.866156 VRAM=38.8GiB | 13.8% done | ETA(epoch): 115s + Train loss: 0.863792 (1265.9s) ETA: 9971min + Val loss: 0.853172 [t_0.0-0.2=1.0777 t_0.2-0.4=1.0152 t_0.4-0.6=0.8539 t_0.6-0.8=0.7287 t_0.8-1.0=0.6562] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 69/499 (14% done) --- + [Epoch 69][10/66] loss=0.948173 avg=0.871837 VRAM=39.0GiB | 13.8% done | ETA(epoch): 1072s + [Epoch 69][20/66] loss=0.663945 avg=0.837294 VRAM=38.8GiB | 13.9% done | ETA(epoch): 882s + [Epoch 69][30/66] loss=0.849692 avg=0.847790 VRAM=39.0GiB | 13.9% done | ETA(epoch): 690s + [Epoch 69][40/66] loss=0.853630 avg=0.845116 VRAM=38.8GiB | 13.9% done | ETA(epoch): 501s + [Epoch 69][50/66] loss=0.778508 avg=0.844031 VRAM=39.0GiB | 14.0% done | ETA(epoch): 308s + [Epoch 69][60/66] loss=0.877093 avg=0.846992 VRAM=38.8GiB | 14.0% done | ETA(epoch): 115s + Train loss: 0.848035 (1269.0s) ETA: 9945min + Val loss: 0.850788 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9886 t_0.4-0.6=0.8559 t_0.6-0.8=0.7288 t_0.8-1.0=0.6555] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 70/499 (14% done) --- + [Epoch 70][10/66] loss=0.853800 avg=0.861394 VRAM=39.0GiB | 14.0% done | ETA(epoch): 1073s + [Epoch 70][20/66] loss=0.869055 avg=0.852365 VRAM=38.8GiB | 14.1% done | ETA(epoch): 883s + [Epoch 70][30/66] loss=0.748300 avg=0.858077 VRAM=39.0GiB | 14.1% done | ETA(epoch): 691s + [Epoch 70][40/66] loss=0.954783 avg=0.866134 VRAM=38.8GiB | 14.1% done | ETA(epoch): 499s + [Epoch 70][50/66] loss=0.953620 avg=0.864714 VRAM=39.0GiB | 14.2% done | ETA(epoch): 307s + [Epoch 70][60/66] loss=0.892687 avg=0.863184 VRAM=38.8GiB | 14.2% done | ETA(epoch): 115s + Train loss: 0.864644 (1266.1s) ETA: 9919min + Val loss: 0.860400 [t_0.0-0.2=1.0839 t_0.2-0.4=1.0013 t_0.4-0.6=0.8233 t_0.6-0.8=0.7388 t_0.8-1.0=0.6479] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 71/499 (14% done) --- + [Epoch 71][10/66] loss=0.747417 avg=0.850982 VRAM=39.0GiB | 14.2% done | ETA(epoch): 1075s + [Epoch 71][20/66] loss=0.922120 avg=0.860484 VRAM=38.8GiB | 14.3% done | ETA(epoch): 887s + [Epoch 71][30/66] loss=0.886491 avg=0.858877 VRAM=39.0GiB | 14.3% done | ETA(epoch): 693s + [Epoch 71][40/66] loss=0.902900 avg=0.859224 VRAM=38.8GiB | 14.3% done | ETA(epoch): 500s + [Epoch 71][50/66] loss=0.798748 avg=0.861313 VRAM=39.0GiB | 14.4% done | ETA(epoch): 308s + [Epoch 71][60/66] loss=0.885570 avg=0.860509 VRAM=38.8GiB | 14.4% done | ETA(epoch): 115s + Train loss: 0.862444 (1268.4s) ETA: 9894min + Val loss: 0.829378 [t_0.0-0.2=1.0783 t_0.2-0.4=1.0201 t_0.4-0.6=0.8140 t_0.6-0.8=0.7388 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 72/499 (14% done) --- + [Epoch 72][10/66] loss=0.980048 avg=0.879754 VRAM=39.0GiB | 14.4% done | ETA(epoch): 1073s + [Epoch 72][20/66] loss=0.900985 avg=0.857041 VRAM=38.8GiB | 14.5% done | ETA(epoch): 882s + [Epoch 72][30/66] loss=0.950832 avg=0.862138 VRAM=39.0GiB | 14.5% done | ETA(epoch): 690s + [Epoch 72][40/66] loss=0.875472 avg=0.854755 VRAM=38.8GiB | 14.5% done | ETA(epoch): 498s + [Epoch 72][50/66] loss=0.893086 avg=0.854321 VRAM=39.0GiB | 14.6% done | ETA(epoch): 307s + [Epoch 72][60/66] loss=0.955452 avg=0.855575 VRAM=38.8GiB | 14.6% done | ETA(epoch): 115s + Train loss: 0.852610 (1265.5s) ETA: 9868min + Val loss: 0.870574 [t_0.0-0.2=1.0695 t_0.2-0.4=0.9779 t_0.4-0.6=0.8789 t_0.6-0.8=0.7115 t_0.8-1.0=0.6274] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 73/499 (15% done) --- + [Epoch 73][10/66] loss=0.860247 avg=0.853533 VRAM=39.0GiB | 14.6% done | ETA(epoch): 1073s + [Epoch 73][20/66] loss=0.908195 avg=0.830678 VRAM=38.8GiB | 14.7% done | ETA(epoch): 882s + [Epoch 73][30/66] loss=0.995164 avg=0.847230 VRAM=39.0GiB | 14.7% done | ETA(epoch): 690s + [Epoch 73][40/66] loss=0.826150 avg=0.849693 VRAM=38.8GiB | 14.7% done | ETA(epoch): 498s + [Epoch 73][50/66] loss=0.818483 avg=0.856568 VRAM=39.0GiB | 14.8% done | ETA(epoch): 307s + [Epoch 73][60/66] loss=0.775000 avg=0.857429 VRAM=38.8GiB | 14.8% done | ETA(epoch): 115s + Train loss: 0.859080 (1268.2s) ETA: 9843min + Val loss: 0.859586 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9882 t_0.4-0.6=0.8662 t_0.6-0.8=0.7117 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 16.7/188.5 GiB (8.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 74/499 (15% done) --- + [Epoch 74][10/66] loss=0.772676 avg=0.880971 VRAM=39.0GiB | 14.8% done | ETA(epoch): 1071s + [Epoch 74][20/66] loss=0.753584 avg=0.876271 VRAM=38.8GiB | 14.9% done | ETA(epoch): 882s + [Epoch 74][30/66] loss=1.052681 avg=0.881956 VRAM=39.0GiB | 14.9% done | ETA(epoch): 690s + [Epoch 74][40/66] loss=0.829832 avg=0.866335 VRAM=38.8GiB | 14.9% done | ETA(epoch): 498s + [Epoch 74][50/66] loss=0.828763 avg=0.870275 VRAM=39.0GiB | 15.0% done | ETA(epoch): 307s + [Epoch 74][60/66] loss=1.036022 avg=0.876559 VRAM=38.8GiB | 15.0% done | ETA(epoch): 115s + Train loss: 0.874652 (1264.4s) ETA: 9817min + Val loss: 0.849235 [t_0.0-0.2=1.0653 t_0.2-0.4=1.0074 t_0.4-0.6=0.8517 t_0.6-0.8=0.7081 t_0.8-1.0=0.6254] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 75/499 (15% done) --- + [Epoch 75][10/66] loss=0.946545 avg=0.846070 VRAM=39.0GiB | 15.0% done | ETA(epoch): 1073s + [Epoch 75][20/66] loss=0.765831 avg=0.850688 VRAM=38.8GiB | 15.1% done | ETA(epoch): 882s + [Epoch 75][30/66] loss=0.881399 avg=0.843606 VRAM=39.0GiB | 15.1% done | ETA(epoch): 690s + [Epoch 75][40/66] loss=0.751509 avg=0.842243 VRAM=38.8GiB | 15.1% done | ETA(epoch): 498s + [Epoch 75][50/66] loss=0.808428 avg=0.847958 VRAM=39.0GiB | 15.2% done | ETA(epoch): 307s + [Epoch 75][60/66] loss=0.891940 avg=0.842760 VRAM=38.8GiB | 15.2% done | ETA(epoch): 115s + Train loss: 0.843211 (1265.4s) ETA: 9792min + Val loss: 0.834935 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0595 t_0.4-0.6=0.8482 t_0.6-0.8=0.7232 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 76/499 (15% done) --- + [Epoch 76][10/66] loss=0.894809 avg=0.822690 VRAM=39.0GiB | 15.2% done | ETA(epoch): 1073s + [Epoch 76][20/66] loss=0.774679 avg=0.832493 VRAM=38.8GiB | 15.3% done | ETA(epoch): 881s + [Epoch 76][30/66] loss=0.920355 avg=0.820704 VRAM=39.0GiB | 15.3% done | ETA(epoch): 690s + [Epoch 76][40/66] loss=0.804840 avg=0.822535 VRAM=38.8GiB | 15.3% done | ETA(epoch): 499s + [Epoch 76][50/66] loss=0.715027 avg=0.829684 VRAM=39.0GiB | 15.4% done | ETA(epoch): 307s + [Epoch 76][60/66] loss=0.795025 avg=0.834829 VRAM=38.8GiB | 15.4% done | ETA(epoch): 115s + Train loss: 0.840547 (1265.6s) ETA: 9766min + Val loss: 0.870391 [t_0.0-0.2=1.0560 t_0.2-0.4=1.0108 t_0.4-0.6=0.8488 t_0.6-0.8=0.7334 t_0.8-1.0=0.6443] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 77/499 (15% done) --- + [Epoch 77][10/66] loss=0.906330 avg=0.859347 VRAM=39.0GiB | 15.4% done | ETA(epoch): 1073s + [Epoch 77][20/66] loss=0.911088 avg=0.866483 VRAM=38.8GiB | 15.5% done | ETA(epoch): 881s + [Epoch 77][30/66] loss=0.920131 avg=0.852528 VRAM=39.0GiB | 15.5% done | ETA(epoch): 690s + [Epoch 77][40/66] loss=0.753228 avg=0.846191 VRAM=38.8GiB | 15.5% done | ETA(epoch): 500s + [Epoch 77][50/66] loss=0.870586 avg=0.845424 VRAM=39.0GiB | 15.6% done | ETA(epoch): 307s + [Epoch 77][60/66] loss=0.845817 avg=0.847856 VRAM=38.8GiB | 15.6% done | ETA(epoch): 115s + Train loss: 0.847837 (1267.1s) ETA: 9741min + Val loss: 0.831798 [t_0.0-0.2=1.0666 t_0.2-0.4=0.9665 t_0.4-0.6=0.8285 t_0.6-0.8=0.7102 t_0.8-1.0=0.6506] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 78/499 (16% done) --- + [Epoch 78][10/66] loss=0.780277 avg=0.844290 VRAM=39.0GiB | 15.6% done | ETA(epoch): 1071s + [Epoch 78][20/66] loss=0.799364 avg=0.858565 VRAM=38.8GiB | 15.7% done | ETA(epoch): 880s + [Epoch 78][30/66] loss=0.823783 avg=0.849193 VRAM=39.0GiB | 15.7% done | ETA(epoch): 689s + [Epoch 78][40/66] loss=0.801225 avg=0.845068 VRAM=38.8GiB | 15.7% done | ETA(epoch): 498s + [Epoch 78][50/66] loss=1.012938 avg=0.850535 VRAM=39.0GiB | 15.8% done | ETA(epoch): 306s + [Epoch 78][60/66] loss=0.927135 avg=0.855334 VRAM=38.8GiB | 15.8% done | ETA(epoch): 115s + Train loss: 0.856083 (1263.8s) ETA: 9716min + Val loss: 0.843906 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9931 t_0.4-0.6=0.8675 t_0.6-0.8=0.6955 t_0.8-1.0=0.6389] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 79/499 (16% done) --- + [Epoch 79][10/66] loss=0.927860 avg=0.863310 VRAM=39.0GiB | 15.8% done | ETA(epoch): 1071s + [Epoch 79][20/66] loss=0.817383 avg=0.853847 VRAM=38.8GiB | 15.9% done | ETA(epoch): 881s + [Epoch 79][30/66] loss=0.882138 avg=0.854088 VRAM=39.0GiB | 15.9% done | ETA(epoch): 690s + [Epoch 79][40/66] loss=0.828750 avg=0.849672 VRAM=38.8GiB | 15.9% done | ETA(epoch): 498s + [Epoch 79][50/66] loss=0.866755 avg=0.847665 VRAM=39.0GiB | 16.0% done | ETA(epoch): 306s + [Epoch 79][60/66] loss=0.865755 avg=0.847726 VRAM=38.8GiB | 16.0% done | ETA(epoch): 115s + Train loss: 0.844080 (1264.6s) ETA: 9690min + Val loss: 0.892266 [t_0.0-0.2=1.0811 t_0.2-0.4=1.0175 t_0.4-0.6=0.7950 t_0.6-0.8=0.7006 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 80/499 (16% done) --- + [Epoch 80][10/66] loss=0.849318 avg=0.860368 VRAM=39.0GiB | 16.0% done | ETA(epoch): 1074s + [Epoch 80][20/66] loss=0.870582 avg=0.869840 VRAM=38.8GiB | 16.1% done | ETA(epoch): 882s + [Epoch 80][30/66] loss=0.715971 avg=0.851595 VRAM=39.0GiB | 16.1% done | ETA(epoch): 690s + [Epoch 80][40/66] loss=0.758650 avg=0.841438 VRAM=38.8GiB | 16.1% done | ETA(epoch): 498s + [Epoch 80][50/66] loss=0.932981 avg=0.851174 VRAM=39.0GiB | 16.2% done | ETA(epoch): 307s + [Epoch 80][60/66] loss=0.945294 avg=0.848214 VRAM=38.8GiB | 16.2% done | ETA(epoch): 115s + Train loss: 0.846691 (1264.7s) ETA: 9665min + Val loss: 0.869403 [t_0.0-0.2=1.0812 t_0.2-0.4=0.9985 t_0.4-0.6=0.8247 t_0.6-0.8=0.7203 t_0.8-1.0=0.6605] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 81/499 (16% done) --- + [Epoch 81][10/66] loss=1.043559 avg=0.901713 VRAM=39.0GiB | 16.2% done | ETA(epoch): 1091s + [Epoch 81][20/66] loss=0.762076 avg=0.900521 VRAM=38.8GiB | 16.3% done | ETA(epoch): 889s + [Epoch 81][30/66] loss=0.828668 avg=0.877227 VRAM=39.0GiB | 16.3% done | ETA(epoch): 694s + [Epoch 81][40/66] loss=0.817564 avg=0.864733 VRAM=38.8GiB | 16.3% done | ETA(epoch): 500s + [Epoch 81][50/66] loss=0.882472 avg=0.859514 VRAM=39.0GiB | 16.4% done | ETA(epoch): 308s + [Epoch 81][60/66] loss=0.909201 avg=0.862294 VRAM=38.8GiB | 16.4% done | ETA(epoch): 115s + Train loss: 0.862794 (1269.8s) ETA: 9640min + Val loss: 0.876859 [t_0.0-0.2=1.0785 t_0.2-0.4=0.9428 t_0.4-0.6=0.8829 t_0.6-0.8=0.7025 t_0.8-1.0=0.6584] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 82/499 (16% done) --- + [Epoch 82][10/66] loss=0.880101 avg=0.857538 VRAM=39.0GiB | 16.4% done | ETA(epoch): 1071s + [Epoch 82][20/66] loss=0.982354 avg=0.847980 VRAM=38.8GiB | 16.5% done | ETA(epoch): 880s + [Epoch 82][30/66] loss=0.888943 avg=0.865131 VRAM=39.0GiB | 16.5% done | ETA(epoch): 689s + [Epoch 82][40/66] loss=0.779449 avg=0.868069 VRAM=38.8GiB | 16.5% done | ETA(epoch): 498s + [Epoch 82][50/66] loss=0.737173 avg=0.866182 VRAM=39.0GiB | 16.6% done | ETA(epoch): 306s + [Epoch 82][60/66] loss=0.857390 avg=0.857023 VRAM=38.8GiB | 16.6% done | ETA(epoch): 115s + Train loss: 0.861682 (1266.1s) ETA: 9615min + Val loss: 0.847317 [t_0.0-0.2=1.0771 t_0.2-0.4=1.0158 t_0.4-0.6=0.8499 t_0.6-0.8=0.7053 t_0.8-1.0=0.6574] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 83/499 (17% done) --- + [Epoch 83][10/66] loss=0.837477 avg=0.909650 VRAM=39.0GiB | 16.6% done | ETA(epoch): 1071s + [Epoch 83][20/66] loss=1.009325 avg=0.898718 VRAM=38.8GiB | 16.7% done | ETA(epoch): 881s + [Epoch 83][30/66] loss=0.913532 avg=0.893495 VRAM=39.0GiB | 16.7% done | ETA(epoch): 690s + [Epoch 83][40/66] loss=0.852349 avg=0.879972 VRAM=38.8GiB | 16.7% done | ETA(epoch): 498s + [Epoch 83][50/66] loss=0.910955 avg=0.887887 VRAM=39.0GiB | 16.8% done | ETA(epoch): 307s + [Epoch 83][60/66] loss=0.819090 avg=0.880885 VRAM=38.8GiB | 16.8% done | ETA(epoch): 115s + Train loss: 0.881361 (1264.7s) ETA: 9590min + Val loss: 0.839416 [t_0.0-0.2=1.0684 t_0.2-0.4=1.0393 t_0.4-0.6=0.8586 t_0.6-0.8=0.7284 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 84/499 (17% done) --- + [Epoch 84][10/66] loss=0.861724 avg=0.826150 VRAM=39.0GiB | 16.8% done | ETA(epoch): 1073s + [Epoch 84][20/66] loss=0.934102 avg=0.849079 VRAM=38.8GiB | 16.9% done | ETA(epoch): 881s + [Epoch 84][30/66] loss=0.855671 avg=0.859513 VRAM=39.0GiB | 16.9% done | ETA(epoch): 690s + [Epoch 84][40/66] loss=0.796790 avg=0.849787 VRAM=38.8GiB | 16.9% done | ETA(epoch): 498s + [Epoch 84][50/66] loss=0.912502 avg=0.857143 VRAM=39.0GiB | 17.0% done | ETA(epoch): 307s + [Epoch 84][60/66] loss=0.822148 avg=0.856888 VRAM=38.8GiB | 17.0% done | ETA(epoch): 115s + Train loss: 0.857641 (1264.5s) ETA: 9565min + Val loss: 0.865115 [t_0.0-0.2=1.0838 t_0.2-0.4=0.9944 t_0.4-0.6=0.8758 t_0.6-0.8=0.7214 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 85/499 (17% done) --- + [Epoch 85][10/66] loss=0.872746 avg=0.825248 VRAM=39.0GiB | 17.0% done | ETA(epoch): 1073s + [Epoch 85][20/66] loss=0.779421 avg=0.839441 VRAM=38.8GiB | 17.1% done | ETA(epoch): 881s + [Epoch 85][30/66] loss=0.760834 avg=0.833672 VRAM=39.0GiB | 17.1% done | ETA(epoch): 690s + [Epoch 85][40/66] loss=0.888712 avg=0.840667 VRAM=38.8GiB | 17.1% done | ETA(epoch): 498s + [Epoch 85][50/66] loss=0.819330 avg=0.841763 VRAM=39.0GiB | 17.2% done | ETA(epoch): 307s + [Epoch 85][60/66] loss=0.725930 avg=0.839339 VRAM=38.8GiB | 17.2% done | ETA(epoch): 115s + Train loss: 0.839471 (1264.4s) ETA: 9540min + Val loss: 0.872490 [t_0.0-0.2=1.0832 t_0.2-0.4=1.0150 t_0.4-0.6=0.8818 t_0.6-0.8=0.7217 t_0.8-1.0=0.6416] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 86/499 (17% done) --- + [Epoch 86][10/66] loss=0.955599 avg=0.872673 VRAM=39.0GiB | 17.2% done | ETA(epoch): 1070s + [Epoch 86][20/66] loss=0.856518 avg=0.866728 VRAM=38.8GiB | 17.3% done | ETA(epoch): 881s + [Epoch 86][30/66] loss=0.763477 avg=0.857798 VRAM=39.0GiB | 17.3% done | ETA(epoch): 689s + [Epoch 86][40/66] loss=0.806503 avg=0.856956 VRAM=38.8GiB | 17.3% done | ETA(epoch): 498s + [Epoch 86][50/66] loss=0.932936 avg=0.858675 VRAM=39.0GiB | 17.4% done | ETA(epoch): 306s + [Epoch 86][60/66] loss=0.849060 avg=0.856338 VRAM=38.8GiB | 17.4% done | ETA(epoch): 115s + Train loss: 0.859642 (1263.5s) ETA: 9515min + Val loss: 0.806842 [t_0.0-0.2=1.0679 t_0.2-0.4=0.9804 t_0.4-0.6=0.8414 t_0.6-0.8=0.7168 t_0.8-1.0=0.6469] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0086 (BEST) + Deleted old checkpoint: checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 87/499 (17% done) --- + [Epoch 87][10/66] loss=0.994041 avg=0.893105 VRAM=39.0GiB | 17.4% done | ETA(epoch): 1072s + [Epoch 87][20/66] loss=0.813691 avg=0.884851 VRAM=38.8GiB | 17.5% done | ETA(epoch): 881s + [Epoch 87][30/66] loss=0.775843 avg=0.861284 VRAM=39.0GiB | 17.5% done | ETA(epoch): 690s + [Epoch 87][40/66] loss=0.949420 avg=0.861429 VRAM=38.8GiB | 17.5% done | ETA(epoch): 498s + [Epoch 87][50/66] loss=0.901724 avg=0.861232 VRAM=39.0GiB | 17.6% done | ETA(epoch): 307s + [Epoch 87][60/66] loss=0.732477 avg=0.853074 VRAM=38.8GiB | 17.6% done | ETA(epoch): 115s + Train loss: 0.849666 (1264.8s) ETA: 9490min + Val loss: 0.878315 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9971 t_0.4-0.6=0.8383 t_0.6-0.8=0.7467 t_0.8-1.0=0.6291] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 88/499 (18% done) --- + [Epoch 88][10/66] loss=0.893426 avg=0.820358 VRAM=39.0GiB | 17.6% done | ETA(epoch): 1093s + [Epoch 88][20/66] loss=0.869507 avg=0.820051 VRAM=38.8GiB | 17.7% done | ETA(epoch): 890s + [Epoch 88][30/66] loss=0.812101 avg=0.817548 VRAM=39.0GiB | 17.7% done | ETA(epoch): 694s + [Epoch 88][40/66] loss=0.734877 avg=0.826795 VRAM=38.8GiB | 17.7% done | ETA(epoch): 500s + [Epoch 88][50/66] loss=0.807992 avg=0.825994 VRAM=39.0GiB | 17.8% done | ETA(epoch): 308s + [Epoch 88][60/66] loss=0.785838 avg=0.838249 VRAM=38.8GiB | 17.8% done | ETA(epoch): 115s + Train loss: 0.835930 (1267.8s) ETA: 9466min + Val loss: 0.839835 [t_0.0-0.2=1.0811 t_0.2-0.4=1.0097 t_0.4-0.6=0.8444 t_0.6-0.8=0.7020 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 89/499 (18% done) --- + [Epoch 89][10/66] loss=0.788476 avg=0.870187 VRAM=39.0GiB | 17.8% done | ETA(epoch): 1073s + [Epoch 89][20/66] loss=0.880690 avg=0.855128 VRAM=38.8GiB | 17.9% done | ETA(epoch): 881s + [Epoch 89][30/66] loss=0.912812 avg=0.863448 VRAM=39.0GiB | 17.9% done | ETA(epoch): 690s + [Epoch 89][40/66] loss=0.837567 avg=0.855805 VRAM=38.8GiB | 17.9% done | ETA(epoch): 499s + [Epoch 89][50/66] loss=0.741240 avg=0.854787 VRAM=39.0GiB | 18.0% done | ETA(epoch): 307s + [Epoch 89][60/66] loss=0.891126 avg=0.858729 VRAM=38.8GiB | 18.0% done | ETA(epoch): 115s + Train loss: 0.860110 (1264.7s) ETA: 9442min + Val loss: 0.821146 [t_0.0-0.2=1.0652 t_0.2-0.4=1.0013 t_0.4-0.6=0.8578 t_0.6-0.8=0.7085 t_0.8-1.0=0.6430] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0089 +[MEM @ epoch 89 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 90/499 (18% done) --- + [Epoch 90][10/66] loss=0.767201 avg=0.882595 VRAM=39.0GiB | 18.0% done | ETA(epoch): 1076s + [Epoch 90][20/66] loss=0.992022 avg=0.868101 VRAM=38.8GiB | 18.1% done | ETA(epoch): 884s + [Epoch 90][30/66] loss=0.820187 avg=0.854610 VRAM=39.0GiB | 18.1% done | ETA(epoch): 692s + [Epoch 90][40/66] loss=0.843481 avg=0.861115 VRAM=38.8GiB | 18.1% done | ETA(epoch): 500s + [Epoch 90][50/66] loss=0.905995 avg=0.851920 VRAM=39.0GiB | 18.2% done | ETA(epoch): 308s + [Epoch 90][60/66] loss=0.962037 avg=0.851701 VRAM=38.8GiB | 18.2% done | ETA(epoch): 116s + Train loss: 0.851750 (1270.6s) ETA: 9417min + Val loss: 0.872317 [t_0.0-0.2=1.0654 t_0.2-0.4=0.9916 t_0.4-0.6=0.8583 t_0.6-0.8=0.7197 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 91/499 (18% done) --- + [Epoch 91][10/66] loss=0.931442 avg=0.854420 VRAM=39.0GiB | 18.2% done | ETA(epoch): 1072s + [Epoch 91][20/66] loss=0.910484 avg=0.868980 VRAM=38.8GiB | 18.3% done | ETA(epoch): 881s + [Epoch 91][30/66] loss=0.914879 avg=0.867272 VRAM=39.0GiB | 18.3% done | ETA(epoch): 689s + [Epoch 91][40/66] loss=0.755340 avg=0.873583 VRAM=38.8GiB | 18.3% done | ETA(epoch): 498s + [Epoch 91][50/66] loss=0.806706 avg=0.869017 VRAM=39.0GiB | 18.4% done | ETA(epoch): 306s + [Epoch 91][60/66] loss=0.784674 avg=0.863424 VRAM=38.8GiB | 18.4% done | ETA(epoch): 115s + Train loss: 0.863155 (1264.3s) ETA: 9393min + Val loss: 0.869781 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9875 t_0.4-0.6=0.8416 t_0.6-0.8=0.7082 t_0.8-1.0=0.6458] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 92/499 (18% done) --- + [Epoch 92][10/66] loss=0.728685 avg=0.825591 VRAM=39.0GiB | 18.4% done | ETA(epoch): 1073s + [Epoch 92][20/66] loss=0.877185 avg=0.826992 VRAM=38.8GiB | 18.5% done | ETA(epoch): 881s + [Epoch 92][30/66] loss=0.839924 avg=0.831355 VRAM=39.0GiB | 18.5% done | ETA(epoch): 690s + [Epoch 92][40/66] loss=0.764678 avg=0.828171 VRAM=38.8GiB | 18.5% done | ETA(epoch): 500s + [Epoch 92][50/66] loss=0.779587 avg=0.834112 VRAM=39.0GiB | 18.6% done | ETA(epoch): 307s + [Epoch 92][60/66] loss=0.725126 avg=0.839550 VRAM=38.8GiB | 18.6% done | ETA(epoch): 115s + Train loss: 0.835239 (1267.1s) ETA: 9368min + Val loss: 0.861139 [t_0.0-0.2=1.0690 t_0.2-0.4=1.0166 t_0.4-0.6=0.8381 t_0.6-0.8=0.7168 t_0.8-1.0=0.6495] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 93/499 (19% done) --- + [Epoch 93][10/66] loss=0.725620 avg=0.812659 VRAM=39.0GiB | 18.6% done | ETA(epoch): 1072s + [Epoch 93][20/66] loss=0.888828 avg=0.841894 VRAM=38.8GiB | 18.7% done | ETA(epoch): 881s + [Epoch 93][30/66] loss=0.879047 avg=0.833488 VRAM=39.0GiB | 18.7% done | ETA(epoch): 690s + [Epoch 93][40/66] loss=0.879852 avg=0.841945 VRAM=38.8GiB | 18.7% done | ETA(epoch): 498s + [Epoch 93][50/66] loss=0.801060 avg=0.843958 VRAM=39.0GiB | 18.8% done | ETA(epoch): 307s + [Epoch 93][60/66] loss=0.936203 avg=0.849133 VRAM=38.8GiB | 18.8% done | ETA(epoch): 115s + Train loss: 0.850583 (1265.7s) ETA: 9344min + Val loss: 0.869951 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9787 t_0.4-0.6=0.8133 t_0.6-0.8=0.7059 t_0.8-1.0=0.6558] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 94/499 (19% done) --- + [Epoch 94][10/66] loss=0.917978 avg=0.860946 VRAM=39.0GiB | 18.8% done | ETA(epoch): 1073s + [Epoch 94][20/66] loss=0.740525 avg=0.848557 VRAM=38.8GiB | 18.9% done | ETA(epoch): 882s + [Epoch 94][30/66] loss=0.970500 avg=0.847678 VRAM=39.0GiB | 18.9% done | ETA(epoch): 690s + [Epoch 94][40/66] loss=0.956085 avg=0.854833 VRAM=38.8GiB | 18.9% done | ETA(epoch): 499s + [Epoch 94][50/66] loss=0.779538 avg=0.857173 VRAM=39.0GiB | 19.0% done | ETA(epoch): 307s + [Epoch 94][60/66] loss=0.905436 avg=0.857087 VRAM=38.8GiB | 19.0% done | ETA(epoch): 115s + Train loss: 0.854285 (1265.7s) ETA: 9319min + Val loss: 0.862087 [t_0.0-0.2=1.0771 t_0.2-0.4=0.9952 t_0.4-0.6=0.8175 t_0.6-0.8=0.7279 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 95/499 (19% done) --- + [Epoch 95][10/66] loss=0.933061 avg=0.871165 VRAM=39.0GiB | 19.0% done | ETA(epoch): 1073s + [Epoch 95][20/66] loss=0.918988 avg=0.861548 VRAM=38.8GiB | 19.1% done | ETA(epoch): 882s + [Epoch 95][30/66] loss=0.908975 avg=0.850270 VRAM=39.0GiB | 19.1% done | ETA(epoch): 690s + [Epoch 95][40/66] loss=0.894691 avg=0.848172 VRAM=38.8GiB | 19.1% done | ETA(epoch): 498s + [Epoch 95][50/66] loss=0.945523 avg=0.858305 VRAM=39.0GiB | 19.2% done | ETA(epoch): 307s + [Epoch 95][60/66] loss=0.904561 avg=0.853404 VRAM=38.8GiB | 19.2% done | ETA(epoch): 115s + Train loss: 0.855958 (1265.5s) ETA: 9295min + Val loss: 0.849611 [t_0.0-0.2=1.0691 t_0.2-0.4=1.0249 t_0.4-0.6=0.8404 t_0.6-0.8=0.6982 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 96/499 (19% done) --- + [Epoch 96][10/66] loss=0.913535 avg=0.890976 VRAM=39.0GiB | 19.2% done | ETA(epoch): 1074s + [Epoch 96][20/66] loss=0.985489 avg=0.867075 VRAM=38.8GiB | 19.3% done | ETA(epoch): 882s + [Epoch 96][30/66] loss=0.820874 avg=0.863018 VRAM=39.0GiB | 19.3% done | ETA(epoch): 690s + [Epoch 96][40/66] loss=0.789251 avg=0.867797 VRAM=38.8GiB | 19.3% done | ETA(epoch): 499s + [Epoch 96][50/66] loss=0.913988 avg=0.867800 VRAM=39.0GiB | 19.4% done | ETA(epoch): 307s + [Epoch 96][60/66] loss=0.761142 avg=0.870496 VRAM=38.8GiB | 19.4% done | ETA(epoch): 115s + Train loss: 0.868574 (1266.1s) ETA: 9270min + Val loss: 0.841371 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0137 t_0.4-0.6=0.8287 t_0.6-0.8=0.7120 t_0.8-1.0=0.6340] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 97/499 (19% done) --- + [Epoch 97][10/66] loss=0.883969 avg=0.852132 VRAM=39.0GiB | 19.4% done | ETA(epoch): 1074s + [Epoch 97][20/66] loss=0.800212 avg=0.856765 VRAM=38.8GiB | 19.5% done | ETA(epoch): 883s + [Epoch 97][30/66] loss=0.825603 avg=0.861581 VRAM=39.0GiB | 19.5% done | ETA(epoch): 691s + [Epoch 97][40/66] loss=0.805163 avg=0.846183 VRAM=38.8GiB | 19.5% done | ETA(epoch): 499s + [Epoch 97][50/66] loss=0.872801 avg=0.851991 VRAM=39.0GiB | 19.6% done | ETA(epoch): 307s + [Epoch 97][60/66] loss=0.796360 avg=0.846336 VRAM=38.8GiB | 19.6% done | ETA(epoch): 116s + Train loss: 0.844634 (1270.1s) ETA: 9246min + Val loss: 0.850262 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9821 t_0.4-0.6=0.8561 t_0.6-0.8=0.6751 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 98/499 (20% done) --- + [Epoch 98][10/66] loss=0.911934 avg=0.853150 VRAM=39.0GiB | 19.6% done | ETA(epoch): 1073s + [Epoch 98][20/66] loss=0.890259 avg=0.851816 VRAM=38.8GiB | 19.7% done | ETA(epoch): 883s + [Epoch 98][30/66] loss=0.876318 avg=0.851626 VRAM=39.0GiB | 19.7% done | ETA(epoch): 692s + [Epoch 98][40/66] loss=0.812078 avg=0.845302 VRAM=38.8GiB | 19.7% done | ETA(epoch): 499s + [Epoch 98][50/66] loss=0.884823 avg=0.843065 VRAM=39.0GiB | 19.8% done | ETA(epoch): 307s + [Epoch 98][60/66] loss=0.890337 avg=0.835414 VRAM=38.8GiB | 19.8% done | ETA(epoch): 115s + Train loss: 0.836223 (1268.0s) ETA: 9222min + Val loss: 0.829551 [t_0.0-0.2=1.0635 t_0.2-0.4=1.0009 t_0.4-0.6=0.8690 t_0.6-0.8=0.7167 t_0.8-1.0=0.6477] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 99/499 (20% done) --- + [Epoch 99][10/66] loss=0.784047 avg=0.878712 VRAM=39.0GiB | 19.8% done | ETA(epoch): 1074s + [Epoch 99][20/66] loss=0.907121 avg=0.859271 VRAM=38.8GiB | 19.9% done | ETA(epoch): 883s + [Epoch 99][30/66] loss=0.885310 avg=0.869123 VRAM=39.0GiB | 19.9% done | ETA(epoch): 691s + [Epoch 99][40/66] loss=0.871564 avg=0.867747 VRAM=38.8GiB | 19.9% done | ETA(epoch): 499s + [Epoch 99][50/66] loss=0.842372 avg=0.866135 VRAM=39.0GiB | 20.0% done | ETA(epoch): 307s + [Epoch 99][60/66] loss=1.005921 avg=0.864746 VRAM=38.8GiB | 20.0% done | ETA(epoch): 115s + Train loss: 0.862202 (1267.5s) ETA: 9198min + Val loss: 0.863059 [t_0.0-0.2=1.0874 t_0.2-0.4=0.9961 t_0.4-0.6=0.8588 t_0.6-0.8=0.7076 t_0.8-1.0=0.6495] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 100/499 (20% done) --- + [MilestoneVis] train_0 step 6600 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 6600 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 6600 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 6600 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 6600 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 6600 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 6600 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 6600 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 6600 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 6600 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 100][10/66] loss=0.804177 avg=0.856927 VRAM=39.0GiB | 20.0% done | ETA(epoch): 6824s + [Epoch 100][20/66] loss=0.744072 avg=0.857355 VRAM=38.8GiB | 20.1% done | ETA(epoch): 3244s + [Epoch 100][30/66] loss=0.755357 avg=0.846704 VRAM=39.0GiB | 20.1% done | ETA(epoch): 1923s + [Epoch 100][40/66] loss=0.708955 avg=0.844488 VRAM=38.8GiB | 20.1% done | ETA(epoch): 1167s + [Epoch 100][50/66] loss=0.902705 avg=0.848367 VRAM=39.0GiB | 20.2% done | ETA(epoch): 636s + [Epoch 100][60/66] loss=0.823421 avg=0.850800 VRAM=38.8GiB | 20.2% done | ETA(epoch): 218s + Train loss: 0.851983 (2294.2s) ETA: 9242min + Val loss: 0.862553 [t_0.0-0.2=1.0790 t_0.2-0.4=0.9769 t_0.4-0.6=0.8370 t_0.6-0.8=0.7307 t_0.8-1.0=0.6532] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 101/499 (20% done) --- + [Epoch 101][10/66] loss=0.846853 avg=0.866246 VRAM=39.0GiB | 20.2% done | ETA(epoch): 1075s + [Epoch 101][20/66] loss=0.799632 avg=0.859920 VRAM=38.8GiB | 20.3% done | ETA(epoch): 883s + [Epoch 101][30/66] loss=0.810104 avg=0.848322 VRAM=39.0GiB | 20.3% done | ETA(epoch): 691s + [Epoch 101][40/66] loss=0.923491 avg=0.850848 VRAM=38.8GiB | 20.3% done | ETA(epoch): 500s + [Epoch 101][50/66] loss=0.950581 avg=0.850864 VRAM=39.0GiB | 20.4% done | ETA(epoch): 307s + [Epoch 101][60/66] loss=0.876423 avg=0.845005 VRAM=38.8GiB | 20.4% done | ETA(epoch): 115s + Train loss: 0.849795 (1268.1s) ETA: 9217min + Val loss: 0.846694 [t_0.0-0.2=1.0906 t_0.2-0.4=0.9929 t_0.4-0.6=0.8508 t_0.6-0.8=0.7073 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 102/499 (20% done) --- + [Epoch 102][10/66] loss=0.759834 avg=0.809658 VRAM=39.0GiB | 20.4% done | ETA(epoch): 1075s + [Epoch 102][20/66] loss=0.869805 avg=0.827221 VRAM=38.8GiB | 20.5% done | ETA(epoch): 884s + [Epoch 102][30/66] loss=0.830819 avg=0.834749 VRAM=39.0GiB | 20.5% done | ETA(epoch): 692s + [Epoch 102][40/66] loss=0.860574 avg=0.837534 VRAM=38.8GiB | 20.5% done | ETA(epoch): 500s + [Epoch 102][50/66] loss=0.863912 avg=0.843274 VRAM=39.0GiB | 20.6% done | ETA(epoch): 307s + [Epoch 102][60/66] loss=0.882129 avg=0.846949 VRAM=38.8GiB | 20.6% done | ETA(epoch): 115s + Train loss: 0.849889 (1267.7s) ETA: 9192min + Val loss: 0.865712 [t_0.0-0.2=1.0695 t_0.2-0.4=0.9921 t_0.4-0.6=0.8350 t_0.6-0.8=0.7037 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 103/499 (21% done) --- + [Epoch 103][10/66] loss=0.923082 avg=0.850624 VRAM=39.0GiB | 20.6% done | ETA(epoch): 1075s + [Epoch 103][20/66] loss=0.849508 avg=0.819490 VRAM=38.8GiB | 20.7% done | ETA(epoch): 883s + [Epoch 103][30/66] loss=0.979253 avg=0.839173 VRAM=39.0GiB | 20.7% done | ETA(epoch): 691s + [Epoch 103][40/66] loss=0.813167 avg=0.841651 VRAM=38.8GiB | 20.7% done | ETA(epoch): 499s + [Epoch 103][50/66] loss=0.774500 avg=0.839317 VRAM=39.0GiB | 20.8% done | ETA(epoch): 307s + [Epoch 103][60/66] loss=0.963701 avg=0.849610 VRAM=38.8GiB | 20.8% done | ETA(epoch): 115s + Train loss: 0.854003 (1267.2s) ETA: 9167min + Val loss: 0.886551 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9781 t_0.4-0.6=0.8616 t_0.6-0.8=0.7167 t_0.8-1.0=0.6611] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 104/499 (21% done) --- + [Epoch 104][10/66] loss=0.795792 avg=0.829151 VRAM=39.0GiB | 20.8% done | ETA(epoch): 1074s + [Epoch 104][20/66] loss=0.871708 avg=0.859061 VRAM=38.8GiB | 20.9% done | ETA(epoch): 883s + [Epoch 104][30/66] loss=0.820258 avg=0.858028 VRAM=39.0GiB | 20.9% done | ETA(epoch): 691s + [Epoch 104][40/66] loss=0.728559 avg=0.847259 VRAM=38.8GiB | 20.9% done | ETA(epoch): 499s + [Epoch 104][50/66] loss=0.895919 avg=0.850745 VRAM=39.0GiB | 21.0% done | ETA(epoch): 307s + [Epoch 104][60/66] loss=0.820086 avg=0.850067 VRAM=38.8GiB | 21.0% done | ETA(epoch): 115s + Train loss: 0.850753 (1266.8s) ETA: 9142min + Val loss: 0.830640 [t_0.0-0.2=1.0906 t_0.2-0.4=1.0108 t_0.4-0.6=0.8402 t_0.6-0.8=0.7162 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 105/499 (21% done) --- + [Epoch 105][10/66] loss=0.751828 avg=0.838322 VRAM=39.0GiB | 21.0% done | ETA(epoch): 1074s + [Epoch 105][20/66] loss=0.808277 avg=0.860778 VRAM=38.8GiB | 21.1% done | ETA(epoch): 882s + [Epoch 105][30/66] loss=0.872522 avg=0.855097 VRAM=39.0GiB | 21.1% done | ETA(epoch): 692s + [Epoch 105][40/66] loss=0.862810 avg=0.848837 VRAM=38.8GiB | 21.1% done | ETA(epoch): 499s + [Epoch 105][50/66] loss=0.872304 avg=0.856555 VRAM=39.0GiB | 21.2% done | ETA(epoch): 307s + [Epoch 105][60/66] loss=0.836101 avg=0.856507 VRAM=38.8GiB | 21.2% done | ETA(epoch): 115s + Train loss: 0.857371 (1267.8s) ETA: 9117min + Val loss: 0.863332 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9600 t_0.4-0.6=0.8436 t_0.6-0.8=0.7361 t_0.8-1.0=0.6302] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 106/499 (21% done) --- + [Epoch 106][10/66] loss=0.758302 avg=0.869181 VRAM=39.0GiB | 21.2% done | ETA(epoch): 1075s + [Epoch 106][20/66] loss=0.938025 avg=0.858943 VRAM=38.8GiB | 21.3% done | ETA(epoch): 884s + [Epoch 106][30/66] loss=0.769409 avg=0.846240 VRAM=39.0GiB | 21.3% done | ETA(epoch): 691s + [Epoch 106][40/66] loss=0.896964 avg=0.848535 VRAM=38.8GiB | 21.3% done | ETA(epoch): 499s + [Epoch 106][50/66] loss=0.979594 avg=0.844459 VRAM=39.0GiB | 21.4% done | ETA(epoch): 307s + [Epoch 106][60/66] loss=0.863085 avg=0.846839 VRAM=38.8GiB | 21.4% done | ETA(epoch): 115s + Train loss: 0.845568 (1268.2s) ETA: 9093min + Val loss: 0.857522 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9845 t_0.4-0.6=0.8453 t_0.6-0.8=0.7237 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 107/499 (21% done) --- + [Epoch 107][10/66] loss=0.760639 avg=0.839193 VRAM=39.0GiB | 21.4% done | ETA(epoch): 1075s + [Epoch 107][20/66] loss=0.818441 avg=0.837542 VRAM=38.8GiB | 21.5% done | ETA(epoch): 883s + [Epoch 107][30/66] loss=0.754853 avg=0.831983 VRAM=39.0GiB | 21.5% done | ETA(epoch): 691s + [Epoch 107][40/66] loss=0.766923 avg=0.842205 VRAM=38.8GiB | 21.5% done | ETA(epoch): 499s + [Epoch 107][50/66] loss=0.760271 avg=0.843066 VRAM=39.0GiB | 21.6% done | ETA(epoch): 307s + [Epoch 107][60/66] loss=0.916370 avg=0.850112 VRAM=38.8GiB | 21.6% done | ETA(epoch): 115s + Train loss: 0.853104 (1267.6s) ETA: 9068min + Val loss: 0.865968 [t_0.0-0.2=1.0801 t_0.2-0.4=1.0055 t_0.4-0.6=0.8301 t_0.6-0.8=0.7178 t_0.8-1.0=0.6568] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 108/499 (22% done) --- + [Epoch 108][10/66] loss=1.001669 avg=0.824921 VRAM=39.0GiB | 21.6% done | ETA(epoch): 1076s + [Epoch 108][20/66] loss=0.832718 avg=0.833758 VRAM=38.8GiB | 21.7% done | ETA(epoch): 883s + [Epoch 108][30/66] loss=0.912673 avg=0.836255 VRAM=39.0GiB | 21.7% done | ETA(epoch): 691s + [Epoch 108][40/66] loss=0.894175 avg=0.844500 VRAM=38.8GiB | 21.7% done | ETA(epoch): 501s + [Epoch 108][50/66] loss=0.781409 avg=0.843846 VRAM=39.0GiB | 21.8% done | ETA(epoch): 308s + [Epoch 108][60/66] loss=0.817904 avg=0.849923 VRAM=38.8GiB | 21.8% done | ETA(epoch): 115s + Train loss: 0.844877 (1270.3s) ETA: 9043min + Val loss: 0.868021 [t_0.0-0.2=1.0862 t_0.2-0.4=0.9785 t_0.4-0.6=0.8700 t_0.6-0.8=0.7265 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 109/499 (22% done) --- + [Epoch 109][10/66] loss=0.899700 avg=0.843878 VRAM=39.0GiB | 21.8% done | ETA(epoch): 1075s + [Epoch 109][20/66] loss=0.727208 avg=0.837162 VRAM=38.8GiB | 21.9% done | ETA(epoch): 884s + [Epoch 109][30/66] loss=0.775491 avg=0.839892 VRAM=39.0GiB | 21.9% done | ETA(epoch): 692s + [Epoch 109][40/66] loss=0.878200 avg=0.846657 VRAM=38.8GiB | 21.9% done | ETA(epoch): 501s + [Epoch 109][50/66] loss=0.763649 avg=0.838279 VRAM=39.0GiB | 22.0% done | ETA(epoch): 308s + [Epoch 109][60/66] loss=0.889554 avg=0.839757 VRAM=38.8GiB | 22.0% done | ETA(epoch): 115s + Train loss: 0.839345 (1270.0s) ETA: 9019min + Val loss: 0.871413 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9835 t_0.4-0.6=0.8378 t_0.6-0.8=0.7214 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 110/499 (22% done) --- + [Epoch 110][10/66] loss=0.807053 avg=0.867101 VRAM=39.0GiB | 22.0% done | ETA(epoch): 1074s + [Epoch 110][20/66] loss=0.828601 avg=0.835962 VRAM=38.8GiB | 22.1% done | ETA(epoch): 883s + [Epoch 110][30/66] loss=0.825012 avg=0.830463 VRAM=39.0GiB | 22.1% done | ETA(epoch): 691s + [Epoch 110][40/66] loss=0.788047 avg=0.835239 VRAM=38.8GiB | 22.1% done | ETA(epoch): 499s + [Epoch 110][50/66] loss=0.908872 avg=0.839098 VRAM=39.0GiB | 22.2% done | ETA(epoch): 307s + [Epoch 110][60/66] loss=0.823280 avg=0.846809 VRAM=38.8GiB | 22.2% done | ETA(epoch): 115s + Train loss: 0.851090 (1267.1s) ETA: 8994min + Val loss: 0.866571 [t_0.0-0.2=1.0709 t_0.2-0.4=1.0150 t_0.4-0.6=0.8492 t_0.6-0.8=0.7129 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 111/499 (22% done) --- + [Epoch 111][10/66] loss=0.916935 avg=0.846175 VRAM=39.0GiB | 22.2% done | ETA(epoch): 1077s + [Epoch 111][20/66] loss=0.899841 avg=0.853912 VRAM=38.8GiB | 22.3% done | ETA(epoch): 884s + [Epoch 111][30/66] loss=0.870258 avg=0.844593 VRAM=39.0GiB | 22.3% done | ETA(epoch): 692s + [Epoch 111][40/66] loss=0.916728 avg=0.852522 VRAM=38.8GiB | 22.3% done | ETA(epoch): 500s + [Epoch 111][50/66] loss=0.844940 avg=0.849425 VRAM=39.0GiB | 22.4% done | ETA(epoch): 307s + [Epoch 111][60/66] loss=0.895121 avg=0.854778 VRAM=38.8GiB | 22.4% done | ETA(epoch): 115s + Train loss: 0.855393 (1268.1s) ETA: 8970min + Val loss: 0.833689 [t_0.0-0.2=1.0793 t_0.2-0.4=0.9795 t_0.4-0.6=0.8726 t_0.6-0.8=0.7036 t_0.8-1.0=0.6273] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 112/499 (22% done) --- + [Epoch 112][10/66] loss=0.852689 avg=0.843412 VRAM=39.0GiB | 22.4% done | ETA(epoch): 1076s + [Epoch 112][20/66] loss=0.825725 avg=0.836413 VRAM=38.8GiB | 22.5% done | ETA(epoch): 884s + [Epoch 112][30/66] loss=0.916488 avg=0.846409 VRAM=39.0GiB | 22.5% done | ETA(epoch): 691s + [Epoch 112][40/66] loss=0.980659 avg=0.853727 VRAM=38.8GiB | 22.5% done | ETA(epoch): 499s + [Epoch 112][50/66] loss=0.960702 avg=0.856868 VRAM=39.0GiB | 22.6% done | ETA(epoch): 307s + [Epoch 112][60/66] loss=0.903907 avg=0.857958 VRAM=38.8GiB | 22.6% done | ETA(epoch): 115s + Train loss: 0.857727 (1268.3s) ETA: 8945min + Val loss: 0.880052 [t_0.0-0.2=1.0669 t_0.2-0.4=1.0300 t_0.4-0.6=0.8231 t_0.6-0.8=0.7446 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 113/499 (23% done) --- + [Epoch 113][10/66] loss=0.848056 avg=0.872621 VRAM=39.0GiB | 22.6% done | ETA(epoch): 1075s + [Epoch 113][20/66] loss=0.717406 avg=0.881346 VRAM=38.8GiB | 22.7% done | ETA(epoch): 883s + [Epoch 113][30/66] loss=0.707656 avg=0.855769 VRAM=39.0GiB | 22.7% done | ETA(epoch): 691s + [Epoch 113][40/66] loss=0.807496 avg=0.858291 VRAM=38.8GiB | 22.7% done | ETA(epoch): 499s + [Epoch 113][50/66] loss=0.750360 avg=0.855010 VRAM=39.0GiB | 22.8% done | ETA(epoch): 307s + [Epoch 113][60/66] loss=0.896762 avg=0.858647 VRAM=38.8GiB | 22.8% done | ETA(epoch): 115s + Train loss: 0.855128 (1268.4s) ETA: 8921min + Val loss: 0.860137 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0100 t_0.4-0.6=0.8422 t_0.6-0.8=0.7046 t_0.8-1.0=0.6434] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 114/499 (23% done) --- + [Epoch 114][10/66] loss=0.930960 avg=0.881486 VRAM=39.0GiB | 22.8% done | ETA(epoch): 1076s + [Epoch 114][20/66] loss=0.844813 avg=0.872790 VRAM=38.8GiB | 22.9% done | ETA(epoch): 883s + [Epoch 114][30/66] loss=0.962173 avg=0.878120 VRAM=39.0GiB | 22.9% done | ETA(epoch): 691s + [Epoch 114][40/66] loss=0.795136 avg=0.859800 VRAM=38.8GiB | 22.9% done | ETA(epoch): 499s + [Epoch 114][50/66] loss=0.858902 avg=0.858791 VRAM=39.0GiB | 23.0% done | ETA(epoch): 307s + [Epoch 114][60/66] loss=0.825698 avg=0.859250 VRAM=38.8GiB | 23.0% done | ETA(epoch): 115s + Train loss: 0.855877 (1267.4s) ETA: 8896min + Val loss: 0.866671 [t_0.0-0.2=1.0680 t_0.2-0.4=0.9534 t_0.4-0.6=0.8809 t_0.6-0.8=0.6993 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 115/499 (23% done) --- + [Epoch 115][10/66] loss=0.786344 avg=0.852282 VRAM=39.0GiB | 23.0% done | ETA(epoch): 1076s + [Epoch 115][20/66] loss=0.809362 avg=0.828471 VRAM=38.8GiB | 23.1% done | ETA(epoch): 884s + [Epoch 115][30/66] loss=0.923146 avg=0.844296 VRAM=39.0GiB | 23.1% done | ETA(epoch): 691s + [Epoch 115][40/66] loss=0.906116 avg=0.842821 VRAM=38.8GiB | 23.1% done | ETA(epoch): 500s + [Epoch 115][50/66] loss=0.823607 avg=0.845164 VRAM=39.0GiB | 23.2% done | ETA(epoch): 307s + [Epoch 115][60/66] loss=0.850641 avg=0.841236 VRAM=38.8GiB | 23.2% done | ETA(epoch): 115s + Train loss: 0.842364 (1268.2s) ETA: 8872min + Val loss: 0.847171 [t_0.0-0.2=1.0549 t_0.2-0.4=1.0206 t_0.4-0.6=0.8374 t_0.6-0.8=0.7115 t_0.8-1.0=0.6445] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 116/499 (23% done) --- + [Epoch 116][10/66] loss=0.764226 avg=0.869692 VRAM=39.0GiB | 23.2% done | ETA(epoch): 1075s + [Epoch 116][20/66] loss=0.865755 avg=0.886541 VRAM=38.8GiB | 23.3% done | ETA(epoch): 882s + [Epoch 116][30/66] loss=0.908956 avg=0.881835 VRAM=39.0GiB | 23.3% done | ETA(epoch): 691s + [Epoch 116][40/66] loss=0.809034 avg=0.877354 VRAM=38.8GiB | 23.3% done | ETA(epoch): 500s + [Epoch 116][50/66] loss=0.820940 avg=0.873276 VRAM=39.0GiB | 23.4% done | ETA(epoch): 308s + [Epoch 116][60/66] loss=0.786705 avg=0.864500 VRAM=38.8GiB | 23.4% done | ETA(epoch): 115s + Train loss: 0.862833 (1269.4s) ETA: 8848min + Val loss: 0.844743 [t_0.0-0.2=1.0375 t_0.2-0.4=0.9974 t_0.4-0.6=0.8480 t_0.6-0.8=0.7180 t_0.8-1.0=0.6314] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 117/499 (23% done) --- + [Epoch 117][10/66] loss=0.902459 avg=0.878398 VRAM=39.0GiB | 23.4% done | ETA(epoch): 1075s + [Epoch 117][20/66] loss=0.857234 avg=0.858025 VRAM=38.8GiB | 23.5% done | ETA(epoch): 884s + [Epoch 117][30/66] loss=0.823202 avg=0.858649 VRAM=39.0GiB | 23.5% done | ETA(epoch): 691s + [Epoch 117][40/66] loss=0.837979 avg=0.855236 VRAM=38.8GiB | 23.5% done | ETA(epoch): 499s + [Epoch 117][50/66] loss=0.900258 avg=0.850651 VRAM=39.0GiB | 23.6% done | ETA(epoch): 307s + [Epoch 117][60/66] loss=0.861663 avg=0.850871 VRAM=38.8GiB | 23.6% done | ETA(epoch): 115s + Train loss: 0.850524 (1267.5s) ETA: 8823min + Val loss: 0.850277 [t_0.0-0.2=1.0584 t_0.2-0.4=0.9955 t_0.4-0.6=0.8514 t_0.6-0.8=0.7165 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 118/499 (24% done) --- + [Epoch 118][10/66] loss=0.907476 avg=0.868020 VRAM=39.0GiB | 23.6% done | ETA(epoch): 1074s + [Epoch 118][20/66] loss=0.913456 avg=0.867741 VRAM=38.8GiB | 23.7% done | ETA(epoch): 883s + [Epoch 118][30/66] loss=0.828329 avg=0.856401 VRAM=39.0GiB | 23.7% done | ETA(epoch): 691s + [Epoch 118][40/66] loss=0.949623 avg=0.850488 VRAM=38.8GiB | 23.7% done | ETA(epoch): 499s + [Epoch 118][50/66] loss=0.783666 avg=0.844596 VRAM=39.0GiB | 23.8% done | ETA(epoch): 307s + [Epoch 118][60/66] loss=0.803341 avg=0.845140 VRAM=38.8GiB | 23.8% done | ETA(epoch): 115s + Train loss: 0.851196 (1267.3s) ETA: 8799min + Val loss: 0.852101 [t_0.0-0.2=1.0629 t_0.2-0.4=1.0159 t_0.4-0.6=0.8484 t_0.6-0.8=0.7138 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 119/499 (24% done) --- + [Epoch 119][10/66] loss=0.886930 avg=0.886216 VRAM=39.0GiB | 23.8% done | ETA(epoch): 1075s + [Epoch 119][20/66] loss=0.888780 avg=0.894338 VRAM=38.8GiB | 23.9% done | ETA(epoch): 893s + [Epoch 119][30/66] loss=0.864113 avg=0.873565 VRAM=39.0GiB | 23.9% done | ETA(epoch): 697s + [Epoch 119][40/66] loss=0.887808 avg=0.870071 VRAM=38.8GiB | 23.9% done | ETA(epoch): 502s + [Epoch 119][50/66] loss=0.679286 avg=0.864441 VRAM=39.0GiB | 24.0% done | ETA(epoch): 309s + [Epoch 119][60/66] loss=0.799242 avg=0.865203 VRAM=38.8GiB | 24.0% done | ETA(epoch): 116s + Train loss: 0.863524 (1272.4s) ETA: 8775min + Val loss: 0.834902 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9900 t_0.4-0.6=0.8276 t_0.6-0.8=0.7213 t_0.8-1.0=0.6682] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 120/499 (24% done) --- + [Epoch 120][10/66] loss=0.977476 avg=0.848127 VRAM=39.0GiB | 24.0% done | ETA(epoch): 1089s + [Epoch 120][20/66] loss=0.901102 avg=0.859039 VRAM=38.8GiB | 24.1% done | ETA(epoch): 889s + [Epoch 120][30/66] loss=0.819311 avg=0.849220 VRAM=39.0GiB | 24.1% done | ETA(epoch): 695s + [Epoch 120][40/66] loss=0.805153 avg=0.844632 VRAM=38.8GiB | 24.1% done | ETA(epoch): 501s + [Epoch 120][50/66] loss=0.878296 avg=0.850844 VRAM=39.0GiB | 24.2% done | ETA(epoch): 308s + [Epoch 120][60/66] loss=0.870254 avg=0.847848 VRAM=38.8GiB | 24.2% done | ETA(epoch): 116s + Train loss: 0.852736 (1273.9s) ETA: 8751min + Val loss: 0.850503 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9997 t_0.4-0.6=0.8568 t_0.6-0.8=0.6963 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 121/499 (24% done) --- + [Epoch 121][10/66] loss=0.818806 avg=0.852067 VRAM=39.0GiB | 24.2% done | ETA(epoch): 1075s + [Epoch 121][20/66] loss=0.824306 avg=0.846283 VRAM=38.8GiB | 24.3% done | ETA(epoch): 883s + [Epoch 121][30/66] loss=0.685218 avg=0.837855 VRAM=39.0GiB | 24.3% done | ETA(epoch): 691s + [Epoch 121][40/66] loss=0.799370 avg=0.827266 VRAM=38.8GiB | 24.3% done | ETA(epoch): 499s + [Epoch 121][50/66] loss=0.858504 avg=0.833916 VRAM=39.0GiB | 24.4% done | ETA(epoch): 307s + [Epoch 121][60/66] loss=0.719538 avg=0.833993 VRAM=38.8GiB | 24.4% done | ETA(epoch): 115s + Train loss: 0.833664 (1269.6s) ETA: 8727min + Val loss: 0.828006 [t_0.0-0.2=1.0563 t_0.2-0.4=1.0102 t_0.4-0.6=0.8510 t_0.6-0.8=0.6865 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 122/499 (24% done) --- + [Epoch 122][10/66] loss=0.829776 avg=0.866280 VRAM=39.0GiB | 24.4% done | ETA(epoch): 1074s + [Epoch 122][20/66] loss=0.876875 avg=0.875978 VRAM=38.8GiB | 24.5% done | ETA(epoch): 884s + [Epoch 122][30/66] loss=0.955004 avg=0.871490 VRAM=39.0GiB | 24.5% done | ETA(epoch): 692s + [Epoch 122][40/66] loss=0.771851 avg=0.865313 VRAM=38.8GiB | 24.5% done | ETA(epoch): 499s + [Epoch 122][50/66] loss=0.904528 avg=0.865567 VRAM=39.0GiB | 24.6% done | ETA(epoch): 308s + [Epoch 122][60/66] loss=0.896451 avg=0.860871 VRAM=38.8GiB | 24.6% done | ETA(epoch): 115s + Train loss: 0.863959 (1270.1s) ETA: 8703min + Val loss: 0.850251 [t_0.0-0.2=1.0852 t_0.2-0.4=1.0003 t_0.4-0.6=0.8266 t_0.6-0.8=0.6966 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 123/499 (25% done) --- + [Epoch 123][10/66] loss=0.971855 avg=0.883742 VRAM=39.0GiB | 24.6% done | ETA(epoch): 1075s + [Epoch 123][20/66] loss=0.792109 avg=0.858514 VRAM=38.8GiB | 24.7% done | ETA(epoch): 883s + [Epoch 123][30/66] loss=1.028407 avg=0.856667 VRAM=39.0GiB | 24.7% done | ETA(epoch): 691s + [Epoch 123][40/66] loss=0.696180 avg=0.850969 VRAM=38.8GiB | 24.7% done | ETA(epoch): 499s + [Epoch 123][50/66] loss=0.957085 avg=0.859899 VRAM=39.0GiB | 24.8% done | ETA(epoch): 307s + [Epoch 123][60/66] loss=0.846704 avg=0.862057 VRAM=38.8GiB | 24.8% done | ETA(epoch): 115s + Train loss: 0.860848 (1267.3s) ETA: 8678min + Val loss: 0.889338 [t_0.0-0.2=1.0759 t_0.2-0.4=1.0023 t_0.4-0.6=0.8392 t_0.6-0.8=0.7097 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 124/499 (25% done) --- + [Epoch 124][10/66] loss=0.860966 avg=0.851828 VRAM=39.0GiB | 24.8% done | ETA(epoch): 1075s + [Epoch 124][20/66] loss=0.882132 avg=0.851620 VRAM=38.8GiB | 24.9% done | ETA(epoch): 883s + [Epoch 124][30/66] loss=0.791450 avg=0.845738 VRAM=39.0GiB | 24.9% done | ETA(epoch): 691s + [Epoch 124][40/66] loss=0.792549 avg=0.852484 VRAM=38.8GiB | 24.9% done | ETA(epoch): 499s + [Epoch 124][50/66] loss=0.958516 avg=0.856283 VRAM=39.0GiB | 25.0% done | ETA(epoch): 307s + [Epoch 124][60/66] loss=0.803426 avg=0.857082 VRAM=38.8GiB | 25.0% done | ETA(epoch): 115s + Train loss: 0.857316 (1267.9s) ETA: 8654min + Val loss: 0.858119 [t_0.0-0.2=1.0635 t_0.2-0.4=0.9982 t_0.4-0.6=0.8447 t_0.6-0.8=0.6906 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 125/499 (25% done) --- + [Epoch 125][10/66] loss=0.989715 avg=0.828775 VRAM=39.0GiB | 25.0% done | ETA(epoch): 1075s + [Epoch 125][20/66] loss=0.815688 avg=0.843320 VRAM=38.8GiB | 25.1% done | ETA(epoch): 883s + [Epoch 125][30/66] loss=0.924196 avg=0.858077 VRAM=39.0GiB | 25.1% done | ETA(epoch): 691s + [Epoch 125][40/66] loss=0.829344 avg=0.858376 VRAM=38.8GiB | 25.1% done | ETA(epoch): 499s + [Epoch 125][50/66] loss=0.954778 avg=0.854247 VRAM=39.0GiB | 25.2% done | ETA(epoch): 307s + [Epoch 125][60/66] loss=0.770117 avg=0.848855 VRAM=38.8GiB | 25.2% done | ETA(epoch): 115s + Train loss: 0.853355 (1267.4s) ETA: 8630min + Val loss: 0.880388 [t_0.0-0.2=1.0770 t_0.2-0.4=1.0169 t_0.4-0.6=0.8468 t_0.6-0.8=0.7131 t_0.8-1.0=0.6410] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 126/499 (25% done) --- + [Epoch 126][10/66] loss=0.641196 avg=0.849307 VRAM=39.0GiB | 25.2% done | ETA(epoch): 1075s + [Epoch 126][20/66] loss=0.757632 avg=0.838234 VRAM=38.8GiB | 25.3% done | ETA(epoch): 888s + [Epoch 126][30/66] loss=0.834674 avg=0.847710 VRAM=39.0GiB | 25.3% done | ETA(epoch): 693s + [Epoch 126][40/66] loss=0.993577 avg=0.854352 VRAM=38.8GiB | 25.3% done | ETA(epoch): 501s + [Epoch 126][50/66] loss=0.867233 avg=0.852613 VRAM=39.0GiB | 25.4% done | ETA(epoch): 308s + [Epoch 126][60/66] loss=0.800418 avg=0.846539 VRAM=38.8GiB | 25.4% done | ETA(epoch): 115s + Train loss: 0.844730 (1269.6s) ETA: 8606min + Val loss: 0.856758 [t_0.0-0.2=1.0693 t_0.2-0.4=1.0015 t_0.4-0.6=0.8614 t_0.6-0.8=0.6992 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 127/499 (25% done) --- + [Epoch 127][10/66] loss=0.903555 avg=0.822776 VRAM=39.0GiB | 25.4% done | ETA(epoch): 1076s + [Epoch 127][20/66] loss=0.783558 avg=0.824853 VRAM=38.8GiB | 25.5% done | ETA(epoch): 884s + [Epoch 127][30/66] loss=0.782189 avg=0.829657 VRAM=39.0GiB | 25.5% done | ETA(epoch): 692s + [Epoch 127][40/66] loss=0.955812 avg=0.824494 VRAM=38.8GiB | 25.5% done | ETA(epoch): 500s + [Epoch 127][50/66] loss=0.823321 avg=0.828605 VRAM=39.0GiB | 25.6% done | ETA(epoch): 308s + [Epoch 127][60/66] loss=0.851488 avg=0.834627 VRAM=38.8GiB | 25.6% done | ETA(epoch): 115s + Train loss: 0.835465 (1268.8s) ETA: 8582min + Val loss: 0.866796 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9955 t_0.4-0.6=0.8335 t_0.6-0.8=0.7238 t_0.8-1.0=0.6237] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 128/499 (26% done) --- + [Epoch 128][10/66] loss=0.745855 avg=0.861855 VRAM=39.0GiB | 25.6% done | ETA(epoch): 1076s + [Epoch 128][20/66] loss=0.910407 avg=0.875959 VRAM=38.8GiB | 25.7% done | ETA(epoch): 884s + [Epoch 128][30/66] loss=0.862325 avg=0.884640 VRAM=39.0GiB | 25.7% done | ETA(epoch): 692s + [Epoch 128][40/66] loss=0.816848 avg=0.881381 VRAM=38.8GiB | 25.7% done | ETA(epoch): 500s + [Epoch 128][50/66] loss=0.902512 avg=0.874551 VRAM=39.0GiB | 25.8% done | ETA(epoch): 308s + [Epoch 128][60/66] loss=0.836618 avg=0.866309 VRAM=38.8GiB | 25.8% done | ETA(epoch): 115s + Train loss: 0.866688 (1268.4s) ETA: 8558min + Val loss: 0.863268 [t_0.0-0.2=1.0757 t_0.2-0.4=1.0156 t_0.4-0.6=0.8686 t_0.6-0.8=0.6919 t_0.8-1.0=0.6601] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 129/499 (26% done) --- + [Epoch 129][10/66] loss=0.778270 avg=0.823751 VRAM=39.0GiB | 25.8% done | ETA(epoch): 1076s + [Epoch 129][20/66] loss=0.785893 avg=0.832733 VRAM=38.8GiB | 25.9% done | ETA(epoch): 885s + [Epoch 129][30/66] loss=0.858681 avg=0.848009 VRAM=39.0GiB | 25.9% done | ETA(epoch): 692s + [Epoch 129][40/66] loss=0.847059 avg=0.839563 VRAM=38.8GiB | 25.9% done | ETA(epoch): 500s + [Epoch 129][50/66] loss=0.778876 avg=0.842552 VRAM=39.0GiB | 26.0% done | ETA(epoch): 308s + [Epoch 129][60/66] loss=0.802168 avg=0.839947 VRAM=38.8GiB | 26.0% done | ETA(epoch): 115s + Train loss: 0.839013 (1268.5s) ETA: 8534min + Val loss: 0.825512 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0136 t_0.4-0.6=0.8127 t_0.6-0.8=0.6988 t_0.8-1.0=0.6494] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 130/499 (26% done) --- + [Epoch 130][10/66] loss=0.886927 avg=0.828859 VRAM=39.0GiB | 26.0% done | ETA(epoch): 1075s + [Epoch 130][20/66] loss=0.945453 avg=0.841794 VRAM=38.8GiB | 26.1% done | ETA(epoch): 883s + [Epoch 130][30/66] loss=0.926144 avg=0.862630 VRAM=39.0GiB | 26.1% done | ETA(epoch): 691s + [Epoch 130][40/66] loss=0.898956 avg=0.865936 VRAM=38.8GiB | 26.1% done | ETA(epoch): 499s + [Epoch 130][50/66] loss=0.892774 avg=0.868494 VRAM=39.0GiB | 26.2% done | ETA(epoch): 307s + [Epoch 130][60/66] loss=0.993802 avg=0.869376 VRAM=38.8GiB | 26.2% done | ETA(epoch): 115s + Train loss: 0.867696 (1267.4s) ETA: 8510min + Val loss: 0.833677 [t_0.0-0.2=1.0527 t_0.2-0.4=1.0201 t_0.4-0.6=0.8445 t_0.6-0.8=0.7326 t_0.8-1.0=0.6335] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 131/499 (26% done) --- + [Epoch 131][10/66] loss=0.857881 avg=0.843442 VRAM=39.0GiB | 26.2% done | ETA(epoch): 1075s + [Epoch 131][20/66] loss=0.841923 avg=0.829465 VRAM=38.8GiB | 26.3% done | ETA(epoch): 884s + [Epoch 131][30/66] loss=0.848973 avg=0.854508 VRAM=39.0GiB | 26.3% done | ETA(epoch): 692s + [Epoch 131][40/66] loss=0.778863 avg=0.850236 VRAM=38.8GiB | 26.3% done | ETA(epoch): 501s + [Epoch 131][50/66] loss=0.916162 avg=0.855187 VRAM=39.0GiB | 26.4% done | ETA(epoch): 308s + [Epoch 131][60/66] loss=0.859443 avg=0.856263 VRAM=38.8GiB | 26.4% done | ETA(epoch): 115s + Train loss: 0.855682 (1270.5s) ETA: 8486min + Val loss: 0.880526 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9952 t_0.4-0.6=0.8234 t_0.6-0.8=0.7129 t_0.8-1.0=0.6490] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 132/499 (26% done) --- + [Epoch 132][10/66] loss=0.760340 avg=0.834923 VRAM=39.0GiB | 26.4% done | ETA(epoch): 1075s + [Epoch 132][20/66] loss=0.983509 avg=0.835494 VRAM=38.8GiB | 26.5% done | ETA(epoch): 884s + [Epoch 132][30/66] loss=0.840346 avg=0.845557 VRAM=39.0GiB | 26.5% done | ETA(epoch): 692s + [Epoch 132][40/66] loss=0.919896 avg=0.845169 VRAM=38.8GiB | 26.5% done | ETA(epoch): 499s + [Epoch 132][50/66] loss=0.731207 avg=0.845237 VRAM=39.0GiB | 26.6% done | ETA(epoch): 307s + [Epoch 132][60/66] loss=0.983301 avg=0.858591 VRAM=38.8GiB | 26.6% done | ETA(epoch): 115s + Train loss: 0.858851 (1268.0s) ETA: 8462min + Val loss: 0.836217 [t_0.0-0.2=1.0608 t_0.2-0.4=0.9740 t_0.4-0.6=0.8587 t_0.6-0.8=0.6988 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 133/499 (27% done) --- + [Epoch 133][10/66] loss=0.713354 avg=0.855367 VRAM=39.0GiB | 26.6% done | ETA(epoch): 1076s + [Epoch 133][20/66] loss=0.796997 avg=0.861743 VRAM=38.8GiB | 26.7% done | ETA(epoch): 884s + [Epoch 133][30/66] loss=0.730050 avg=0.851363 VRAM=39.0GiB | 26.7% done | ETA(epoch): 692s + [Epoch 133][40/66] loss=0.853266 avg=0.850425 VRAM=38.8GiB | 26.7% done | ETA(epoch): 500s + [Epoch 133][50/66] loss=1.010185 avg=0.850716 VRAM=39.0GiB | 26.8% done | ETA(epoch): 307s + [Epoch 133][60/66] loss=0.856374 avg=0.853620 VRAM=38.8GiB | 26.8% done | ETA(epoch): 115s + Train loss: 0.853817 (1268.5s) ETA: 8438min + Val loss: 0.846846 [t_0.0-0.2=1.0602 t_0.2-0.4=1.0220 t_0.4-0.6=0.8572 t_0.6-0.8=0.7175 t_0.8-1.0=0.6452] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 134/499 (27% done) --- + [Epoch 134][10/66] loss=0.878929 avg=0.795070 VRAM=39.0GiB | 26.8% done | ETA(epoch): 1075s + [Epoch 134][20/66] loss=0.863167 avg=0.812899 VRAM=38.8GiB | 26.9% done | ETA(epoch): 883s + [Epoch 134][30/66] loss=0.767857 avg=0.811901 VRAM=39.0GiB | 26.9% done | ETA(epoch): 691s + [Epoch 134][40/66] loss=0.879586 avg=0.812337 VRAM=38.8GiB | 26.9% done | ETA(epoch): 499s + [Epoch 134][50/66] loss=0.820307 avg=0.831910 VRAM=39.0GiB | 27.0% done | ETA(epoch): 307s + [Epoch 134][60/66] loss=0.836512 avg=0.841806 VRAM=38.8GiB | 27.0% done | ETA(epoch): 115s + Train loss: 0.844141 (1267.2s) ETA: 8414min + Val loss: 0.840763 [t_0.0-0.2=1.0860 t_0.2-0.4=0.9835 t_0.4-0.6=0.8429 t_0.6-0.8=0.6925 t_0.8-1.0=0.6331] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 135/499 (27% done) --- + [Epoch 135][10/66] loss=0.892047 avg=0.847496 VRAM=39.0GiB | 27.0% done | ETA(epoch): 1097s + [Epoch 135][20/66] loss=0.710606 avg=0.840071 VRAM=38.8GiB | 27.1% done | ETA(epoch): 892s + [Epoch 135][30/66] loss=0.734758 avg=0.841335 VRAM=39.0GiB | 27.1% done | ETA(epoch): 696s + [Epoch 135][40/66] loss=0.750583 avg=0.845293 VRAM=38.8GiB | 27.1% done | ETA(epoch): 501s + [Epoch 135][50/66] loss=0.809885 avg=0.844021 VRAM=39.0GiB | 27.2% done | ETA(epoch): 308s + [Epoch 135][60/66] loss=0.894676 avg=0.850093 VRAM=38.8GiB | 27.2% done | ETA(epoch): 116s + Train loss: 0.851124 (1271.2s) ETA: 8390min + Val loss: 0.860508 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9792 t_0.4-0.6=0.8199 t_0.6-0.8=0.7130 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 17.4/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 136/499 (27% done) --- + [Epoch 136][10/66] loss=0.949226 avg=0.847706 VRAM=39.0GiB | 27.2% done | ETA(epoch): 1075s + [Epoch 136][20/66] loss=0.963510 avg=0.860464 VRAM=38.8GiB | 27.3% done | ETA(epoch): 883s + [Epoch 136][30/66] loss=0.870965 avg=0.886287 VRAM=39.0GiB | 27.3% done | ETA(epoch): 691s + [Epoch 136][40/66] loss=0.828817 avg=0.872909 VRAM=38.8GiB | 27.3% done | ETA(epoch): 499s + [Epoch 136][50/66] loss=0.751328 avg=0.870720 VRAM=39.0GiB | 27.4% done | ETA(epoch): 307s + [Epoch 136][60/66] loss=0.937213 avg=0.863477 VRAM=38.8GiB | 27.4% done | ETA(epoch): 115s + Train loss: 0.864089 (1267.7s) ETA: 8366min + Val loss: 0.848725 [t_0.0-0.2=1.0667 t_0.2-0.4=1.0014 t_0.4-0.6=0.8529 t_0.6-0.8=0.6995 t_0.8-1.0=0.6528] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 137/499 (27% done) --- + [Epoch 137][10/66] loss=0.893269 avg=0.869888 VRAM=39.0GiB | 27.4% done | ETA(epoch): 1076s + [Epoch 137][20/66] loss=0.869905 avg=0.854605 VRAM=38.8GiB | 27.5% done | ETA(epoch): 884s + [Epoch 137][30/66] loss=0.777482 avg=0.863240 VRAM=39.0GiB | 27.5% done | ETA(epoch): 692s + [Epoch 137][40/66] loss=0.818964 avg=0.857176 VRAM=38.8GiB | 27.5% done | ETA(epoch): 500s + [Epoch 137][50/66] loss=0.940608 avg=0.859071 VRAM=39.0GiB | 27.6% done | ETA(epoch): 307s + [Epoch 137][60/66] loss=0.923853 avg=0.851418 VRAM=38.8GiB | 27.6% done | ETA(epoch): 115s + Train loss: 0.854974 (1267.9s) ETA: 8342min + Val loss: 0.869103 [t_0.0-0.2=1.0690 t_0.2-0.4=0.9673 t_0.4-0.6=0.8856 t_0.6-0.8=0.7143 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 138/499 (28% done) --- + [Epoch 138][10/66] loss=0.919253 avg=0.851178 VRAM=39.0GiB | 27.6% done | ETA(epoch): 1075s + [Epoch 138][20/66] loss=0.900421 avg=0.836277 VRAM=38.8GiB | 27.7% done | ETA(epoch): 884s + [Epoch 138][30/66] loss=0.718219 avg=0.836458 VRAM=39.0GiB | 27.7% done | ETA(epoch): 692s + [Epoch 138][40/66] loss=0.911601 avg=0.845629 VRAM=38.8GiB | 27.7% done | ETA(epoch): 499s + [Epoch 138][50/66] loss=0.856315 avg=0.851081 VRAM=39.0GiB | 27.8% done | ETA(epoch): 308s + [Epoch 138][60/66] loss=0.736735 avg=0.852161 VRAM=38.8GiB | 27.8% done | ETA(epoch): 116s + Train loss: 0.850018 (1270.3s) ETA: 8319min + Val loss: 0.809739 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9883 t_0.4-0.6=0.8451 t_0.6-0.8=0.7024 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 139/499 (28% done) --- + [Epoch 139][10/66] loss=0.790470 avg=0.854419 VRAM=39.0GiB | 27.8% done | ETA(epoch): 1074s + [Epoch 139][20/66] loss=0.972273 avg=0.859797 VRAM=38.8GiB | 27.9% done | ETA(epoch): 883s + [Epoch 139][30/66] loss=0.903834 avg=0.871741 VRAM=39.0GiB | 27.9% done | ETA(epoch): 691s + [Epoch 139][40/66] loss=0.820424 avg=0.862904 VRAM=38.8GiB | 27.9% done | ETA(epoch): 502s + [Epoch 139][50/66] loss=1.023295 avg=0.863149 VRAM=39.0GiB | 28.0% done | ETA(epoch): 308s + [Epoch 139][60/66] loss=0.792039 avg=0.862115 VRAM=38.8GiB | 28.0% done | ETA(epoch): 116s + Train loss: 0.858517 (1271.0s) ETA: 8295min + Val loss: 0.834392 [t_0.0-0.2=1.0871 t_0.2-0.4=0.9936 t_0.4-0.6=0.8273 t_0.6-0.8=0.7136 t_0.8-1.0=0.6329] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 140/499 (28% done) --- + [Epoch 140][10/66] loss=0.948664 avg=0.888686 VRAM=39.0GiB | 28.0% done | ETA(epoch): 1074s + [Epoch 140][20/66] loss=0.867716 avg=0.854183 VRAM=38.8GiB | 28.1% done | ETA(epoch): 883s + [Epoch 140][30/66] loss=0.820722 avg=0.864838 VRAM=39.0GiB | 28.1% done | ETA(epoch): 691s + [Epoch 140][40/66] loss=0.804824 avg=0.868522 VRAM=38.8GiB | 28.1% done | ETA(epoch): 499s + [Epoch 140][50/66] loss=0.730382 avg=0.877630 VRAM=39.0GiB | 28.2% done | ETA(epoch): 307s + [Epoch 140][60/66] loss=0.816502 avg=0.866802 VRAM=38.8GiB | 28.2% done | ETA(epoch): 115s + Train loss: 0.861200 (1267.9s) ETA: 8271min + Val loss: 0.894965 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9989 t_0.4-0.6=0.8510 t_0.6-0.8=0.6931 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 141/499 (28% done) --- + [Epoch 141][10/66] loss=0.876951 avg=0.828740 VRAM=39.0GiB | 28.2% done | ETA(epoch): 1075s + [Epoch 141][20/66] loss=0.918387 avg=0.849549 VRAM=38.8GiB | 28.3% done | ETA(epoch): 883s + [Epoch 141][30/66] loss=0.912527 avg=0.858976 VRAM=39.0GiB | 28.3% done | ETA(epoch): 691s + [Epoch 141][40/66] loss=0.738081 avg=0.855215 VRAM=38.8GiB | 28.3% done | ETA(epoch): 499s + [Epoch 141][50/66] loss=0.908763 avg=0.854974 VRAM=39.0GiB | 28.4% done | ETA(epoch): 308s + [Epoch 141][60/66] loss=0.880725 avg=0.852032 VRAM=38.8GiB | 28.4% done | ETA(epoch): 115s + Train loss: 0.853958 (1268.6s) ETA: 8247min + Val loss: 0.849171 [t_0.0-0.2=1.0875 t_0.2-0.4=1.0097 t_0.4-0.6=0.8579 t_0.6-0.8=0.7113 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 142/499 (28% done) --- + [Epoch 142][10/66] loss=0.940206 avg=0.815703 VRAM=39.0GiB | 28.4% done | ETA(epoch): 1074s + [Epoch 142][20/66] loss=0.939119 avg=0.833328 VRAM=38.8GiB | 28.5% done | ETA(epoch): 882s + [Epoch 142][30/66] loss=0.794648 avg=0.823033 VRAM=39.0GiB | 28.5% done | ETA(epoch): 691s + [Epoch 142][40/66] loss=0.914537 avg=0.836388 VRAM=38.8GiB | 28.5% done | ETA(epoch): 499s + [Epoch 142][50/66] loss=0.857784 avg=0.832614 VRAM=39.0GiB | 28.6% done | ETA(epoch): 307s + [Epoch 142][60/66] loss=0.842475 avg=0.839229 VRAM=38.8GiB | 28.6% done | ETA(epoch): 115s + Train loss: 0.844826 (1267.0s) ETA: 8223min + Val loss: 0.853349 [t_0.0-0.2=1.0882 t_0.2-0.4=0.9836 t_0.4-0.6=0.8445 t_0.6-0.8=0.7539 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 143/499 (29% done) --- + [Epoch 143][10/66] loss=1.007013 avg=0.853111 VRAM=39.0GiB | 28.6% done | ETA(epoch): 1092s + [Epoch 143][20/66] loss=0.810502 avg=0.860705 VRAM=38.8GiB | 28.7% done | ETA(epoch): 891s + [Epoch 143][30/66] loss=0.716324 avg=0.863473 VRAM=39.0GiB | 28.7% done | ETA(epoch): 695s + [Epoch 143][40/66] loss=0.975418 avg=0.855967 VRAM=38.8GiB | 28.7% done | ETA(epoch): 501s + [Epoch 143][50/66] loss=0.875871 avg=0.856425 VRAM=39.0GiB | 28.8% done | ETA(epoch): 308s + [Epoch 143][60/66] loss=0.815729 avg=0.852830 VRAM=38.8GiB | 28.8% done | ETA(epoch): 116s + Train loss: 0.852003 (1272.7s) ETA: 8200min + Val loss: 0.866363 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0092 t_0.4-0.6=0.8520 t_0.6-0.8=0.6959 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 17.8/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 144/499 (29% done) --- + [Epoch 144][10/66] loss=0.784218 avg=0.824604 VRAM=39.0GiB | 28.8% done | ETA(epoch): 1074s + [Epoch 144][20/66] loss=0.797053 avg=0.844306 VRAM=38.8GiB | 28.9% done | ETA(epoch): 883s + [Epoch 144][30/66] loss=0.885237 avg=0.848607 VRAM=39.0GiB | 28.9% done | ETA(epoch): 691s + [Epoch 144][40/66] loss=0.728571 avg=0.840699 VRAM=38.8GiB | 28.9% done | ETA(epoch): 499s + [Epoch 144][50/66] loss=0.896378 avg=0.835341 VRAM=39.0GiB | 29.0% done | ETA(epoch): 307s + [Epoch 144][60/66] loss=0.785885 avg=0.835679 VRAM=38.8GiB | 29.0% done | ETA(epoch): 115s + Train loss: 0.839850 (1269.2s) ETA: 8176min + Val loss: 0.846144 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9800 t_0.4-0.6=0.8615 t_0.6-0.8=0.7022 t_0.8-1.0=0.6312] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 145/499 (29% done) --- + [Epoch 145][10/66] loss=0.805421 avg=0.872459 VRAM=39.0GiB | 29.0% done | ETA(epoch): 1075s + [Epoch 145][20/66] loss=0.716269 avg=0.859373 VRAM=38.8GiB | 29.1% done | ETA(epoch): 884s + [Epoch 145][30/66] loss=0.965377 avg=0.874187 VRAM=39.0GiB | 29.1% done | ETA(epoch): 692s + [Epoch 145][40/66] loss=0.920182 avg=0.864874 VRAM=38.8GiB | 29.1% done | ETA(epoch): 500s + [Epoch 145][50/66] loss=0.809632 avg=0.852488 VRAM=39.0GiB | 29.2% done | ETA(epoch): 307s + [Epoch 145][60/66] loss=0.881783 avg=0.847601 VRAM=38.8GiB | 29.2% done | ETA(epoch): 115s + Train loss: 0.848925 (1268.5s) ETA: 8152min + Val loss: 0.859137 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9959 t_0.4-0.6=0.8437 t_0.6-0.8=0.6979 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 146/499 (29% done) --- + [Epoch 146][10/66] loss=0.745014 avg=0.841443 VRAM=39.0GiB | 29.2% done | ETA(epoch): 1075s + [Epoch 146][20/66] loss=0.770802 avg=0.855891 VRAM=38.8GiB | 29.3% done | ETA(epoch): 884s + [Epoch 146][30/66] loss=0.841335 avg=0.848260 VRAM=39.0GiB | 29.3% done | ETA(epoch): 692s + [Epoch 146][40/66] loss=0.734724 avg=0.841497 VRAM=38.8GiB | 29.3% done | ETA(epoch): 499s + [Epoch 146][50/66] loss=0.804644 avg=0.842525 VRAM=39.0GiB | 29.4% done | ETA(epoch): 307s + [Epoch 146][60/66] loss=0.893195 avg=0.845173 VRAM=38.8GiB | 29.4% done | ETA(epoch): 115s + Train loss: 0.847440 (1268.1s) ETA: 8128min + Val loss: 0.862572 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9655 t_0.4-0.6=0.8388 t_0.6-0.8=0.7273 t_0.8-1.0=0.6364] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 147/499 (29% done) --- + [Epoch 147][10/66] loss=0.847722 avg=0.819850 VRAM=39.0GiB | 29.4% done | ETA(epoch): 1074s + [Epoch 147][20/66] loss=0.922290 avg=0.820887 VRAM=38.8GiB | 29.5% done | ETA(epoch): 883s + [Epoch 147][30/66] loss=0.817927 avg=0.829767 VRAM=39.0GiB | 29.5% done | ETA(epoch): 691s + [Epoch 147][40/66] loss=0.852517 avg=0.827056 VRAM=38.8GiB | 29.5% done | ETA(epoch): 499s + [Epoch 147][50/66] loss=0.846113 avg=0.833070 VRAM=39.0GiB | 29.6% done | ETA(epoch): 307s + [Epoch 147][60/66] loss=0.951509 avg=0.833952 VRAM=38.8GiB | 29.6% done | ETA(epoch): 115s + Train loss: 0.831335 (1267.1s) ETA: 8105min + Val loss: 0.838487 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9890 t_0.4-0.6=0.8336 t_0.6-0.8=0.7229 t_0.8-1.0=0.6333] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 148/499 (30% done) --- + [Epoch 148][10/66] loss=0.889264 avg=0.891379 VRAM=39.0GiB | 29.6% done | ETA(epoch): 1078s + [Epoch 148][20/66] loss=0.963634 avg=0.865250 VRAM=38.8GiB | 29.7% done | ETA(epoch): 884s + [Epoch 148][30/66] loss=0.865209 avg=0.850093 VRAM=39.0GiB | 29.7% done | ETA(epoch): 692s + [Epoch 148][40/66] loss=0.723419 avg=0.854920 VRAM=38.8GiB | 29.7% done | ETA(epoch): 500s + [Epoch 148][50/66] loss=0.928880 avg=0.857607 VRAM=39.0GiB | 29.8% done | ETA(epoch): 307s + [Epoch 148][60/66] loss=1.082087 avg=0.862792 VRAM=38.8GiB | 29.8% done | ETA(epoch): 115s + Train loss: 0.860209 (1268.0s) ETA: 8081min + Val loss: 0.831646 [t_0.0-0.2=1.0614 t_0.2-0.4=0.9841 t_0.4-0.6=0.8536 t_0.6-0.8=0.7085 t_0.8-1.0=0.6420] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 149/499 (30% done) --- + [Epoch 149][10/66] loss=0.785718 avg=0.891085 VRAM=39.0GiB | 29.8% done | ETA(epoch): 1074s + [Epoch 149][20/66] loss=0.942487 avg=0.878797 VRAM=38.8GiB | 29.9% done | ETA(epoch): 882s + [Epoch 149][30/66] loss=0.886860 avg=0.882600 VRAM=39.0GiB | 29.9% done | ETA(epoch): 691s + [Epoch 149][40/66] loss=0.872178 avg=0.876756 VRAM=38.8GiB | 29.9% done | ETA(epoch): 499s + [Epoch 149][50/66] loss=0.794636 avg=0.874230 VRAM=39.0GiB | 30.0% done | ETA(epoch): 307s + [Epoch 149][60/66] loss=0.901381 avg=0.877770 VRAM=38.8GiB | 30.0% done | ETA(epoch): 115s + Train loss: 0.878006 (1266.9s) ETA: 8057min + Val loss: 0.835006 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9740 t_0.4-0.6=0.8397 t_0.6-0.8=0.7207 t_0.8-1.0=0.6249] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 150/499 (30% done) --- + [MilestoneVis] train_0 step 9900 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9900 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9900 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9900 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9900 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9900 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9900 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9900 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9900 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9900 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/66] loss=0.957451 avg=0.884364 VRAM=39.0GiB | 30.0% done | ETA(epoch): 6846s + [Epoch 150][20/66] loss=0.864592 avg=0.865774 VRAM=38.8GiB | 30.1% done | ETA(epoch): 3266s + [Epoch 150][30/66] loss=0.828386 avg=0.847064 VRAM=39.0GiB | 30.1% done | ETA(epoch): 1935s + [Epoch 150][40/66] loss=0.741526 avg=0.844059 VRAM=38.8GiB | 30.1% done | ETA(epoch): 1173s + [Epoch 150][50/66] loss=0.788111 avg=0.842338 VRAM=39.0GiB | 30.2% done | ETA(epoch): 639s + [Epoch 150][60/66] loss=0.943114 avg=0.850794 VRAM=38.8GiB | 30.2% done | ETA(epoch): 219s + Train loss: 0.850188 (2304.2s) ETA: 8073min + Val loss: 0.854280 [t_0.0-0.2=1.0625 t_0.2-0.4=1.0255 t_0.4-0.6=0.8207 t_0.6-0.8=0.6957 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 151/499 (30% done) --- + [Epoch 151][10/66] loss=0.829058 avg=0.852292 VRAM=39.0GiB | 30.2% done | ETA(epoch): 1077s + [Epoch 151][20/66] loss=0.802483 avg=0.851718 VRAM=38.8GiB | 30.3% done | ETA(epoch): 884s + [Epoch 151][30/66] loss=0.882716 avg=0.852570 VRAM=39.0GiB | 30.3% done | ETA(epoch): 692s + [Epoch 151][40/66] loss=0.802789 avg=0.852229 VRAM=38.8GiB | 30.3% done | ETA(epoch): 500s + [Epoch 151][50/66] loss=0.994278 avg=0.854872 VRAM=39.0GiB | 30.4% done | ETA(epoch): 307s + [Epoch 151][60/66] loss=0.752639 avg=0.851417 VRAM=38.8GiB | 30.4% done | ETA(epoch): 115s + Train loss: 0.848774 (1267.8s) ETA: 8049min + Val loss: 0.878739 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9844 t_0.4-0.6=0.8414 t_0.6-0.8=0.7114 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 152/499 (30% done) --- + [Epoch 152][10/66] loss=0.837759 avg=0.852818 VRAM=39.0GiB | 30.4% done | ETA(epoch): 1075s + [Epoch 152][20/66] loss=0.945987 avg=0.847556 VRAM=38.8GiB | 30.5% done | ETA(epoch): 883s + [Epoch 152][30/66] loss=0.700045 avg=0.850425 VRAM=39.0GiB | 30.5% done | ETA(epoch): 691s + [Epoch 152][40/66] loss=0.691533 avg=0.843724 VRAM=38.8GiB | 30.5% done | ETA(epoch): 499s + [Epoch 152][50/66] loss=0.769646 avg=0.847019 VRAM=39.0GiB | 30.6% done | ETA(epoch): 307s + [Epoch 152][60/66] loss=0.800878 avg=0.844636 VRAM=38.8GiB | 30.6% done | ETA(epoch): 115s + Train loss: 0.847413 (1268.0s) ETA: 8025min + Val loss: 0.820953 [t_0.0-0.2=1.0708 t_0.2-0.4=0.9613 t_0.4-0.6=0.8603 t_0.6-0.8=0.7030 t_0.8-1.0=0.6356] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 153/499 (31% done) --- + [Epoch 153][10/66] loss=0.699375 avg=0.852575 VRAM=39.0GiB | 30.6% done | ETA(epoch): 1074s + [Epoch 153][20/66] loss=0.904599 avg=0.875298 VRAM=38.8GiB | 30.7% done | ETA(epoch): 883s + [Epoch 153][30/66] loss=0.833155 avg=0.871181 VRAM=39.0GiB | 30.7% done | ETA(epoch): 691s + [Epoch 153][40/66] loss=0.827774 avg=0.860855 VRAM=38.8GiB | 30.7% done | ETA(epoch): 499s + [Epoch 153][50/66] loss=0.888784 avg=0.862574 VRAM=39.0GiB | 30.8% done | ETA(epoch): 307s + [Epoch 153][60/66] loss=0.717773 avg=0.861697 VRAM=38.8GiB | 30.8% done | ETA(epoch): 115s + Train loss: 0.862635 (1270.1s) ETA: 8001min + Val loss: 0.844745 [t_0.0-0.2=1.0808 t_0.2-0.4=1.0028 t_0.4-0.6=0.8384 t_0.6-0.8=0.7110 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 154/499 (31% done) --- + [Epoch 154][10/66] loss=0.871471 avg=0.870985 VRAM=39.0GiB | 30.8% done | ETA(epoch): 1076s + [Epoch 154][20/66] loss=0.889417 avg=0.863568 VRAM=38.8GiB | 30.9% done | ETA(epoch): 885s + [Epoch 154][30/66] loss=0.868095 avg=0.856787 VRAM=39.0GiB | 30.9% done | ETA(epoch): 692s + [Epoch 154][40/66] loss=0.807110 avg=0.858074 VRAM=38.8GiB | 30.9% done | ETA(epoch): 500s + [Epoch 154][50/66] loss=0.910923 avg=0.856609 VRAM=39.0GiB | 31.0% done | ETA(epoch): 308s + [Epoch 154][60/66] loss=0.772875 avg=0.858887 VRAM=38.8GiB | 31.0% done | ETA(epoch): 115s + Train loss: 0.858413 (1269.7s) ETA: 7977min + Val loss: 0.837708 [t_0.0-0.2=1.0635 t_0.2-0.4=0.9777 t_0.4-0.6=0.8599 t_0.6-0.8=0.7053 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 155/499 (31% done) --- + [Epoch 155][10/66] loss=0.757082 avg=0.841473 VRAM=39.0GiB | 31.0% done | ETA(epoch): 1076s + [Epoch 155][20/66] loss=0.908144 avg=0.853324 VRAM=38.8GiB | 31.1% done | ETA(epoch): 883s + [Epoch 155][30/66] loss=0.833203 avg=0.853029 VRAM=39.0GiB | 31.1% done | ETA(epoch): 692s + [Epoch 155][40/66] loss=0.902108 avg=0.851868 VRAM=38.8GiB | 31.1% done | ETA(epoch): 500s + [Epoch 155][50/66] loss=0.776918 avg=0.845015 VRAM=39.0GiB | 31.2% done | ETA(epoch): 308s + [Epoch 155][60/66] loss=0.770991 avg=0.847050 VRAM=38.8GiB | 31.2% done | ETA(epoch): 115s + Train loss: 0.847192 (1268.6s) ETA: 7953min + Val loss: 0.816412 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9734 t_0.4-0.6=0.8655 t_0.6-0.8=0.7174 t_0.8-1.0=0.6322] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 156/499 (31% done) --- + [Epoch 156][10/66] loss=0.929902 avg=0.893812 VRAM=39.0GiB | 31.2% done | ETA(epoch): 1075s + [Epoch 156][20/66] loss=0.850166 avg=0.855663 VRAM=38.8GiB | 31.3% done | ETA(epoch): 883s + [Epoch 156][30/66] loss=0.813871 avg=0.854942 VRAM=39.0GiB | 31.3% done | ETA(epoch): 692s + [Epoch 156][40/66] loss=0.940041 avg=0.851421 VRAM=38.8GiB | 31.3% done | ETA(epoch): 499s + [Epoch 156][50/66] loss=0.890100 avg=0.857058 VRAM=39.0GiB | 31.4% done | ETA(epoch): 307s + [Epoch 156][60/66] loss=0.948231 avg=0.853677 VRAM=38.8GiB | 31.4% done | ETA(epoch): 115s + Train loss: 0.855790 (1267.8s) ETA: 7929min + Val loss: 0.884890 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9976 t_0.4-0.6=0.8248 t_0.6-0.8=0.7110 t_0.8-1.0=0.6249] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 157/499 (31% done) --- + [Epoch 157][10/66] loss=0.814962 avg=0.866890 VRAM=39.0GiB | 31.4% done | ETA(epoch): 1078s + [Epoch 157][20/66] loss=0.822736 avg=0.859460 VRAM=38.8GiB | 31.5% done | ETA(epoch): 893s + [Epoch 157][30/66] loss=0.836464 avg=0.856221 VRAM=39.0GiB | 31.5% done | ETA(epoch): 697s + [Epoch 157][40/66] loss=0.868958 avg=0.849359 VRAM=38.8GiB | 31.5% done | ETA(epoch): 502s + [Epoch 157][50/66] loss=0.909713 avg=0.839467 VRAM=39.0GiB | 31.6% done | ETA(epoch): 309s + [Epoch 157][60/66] loss=0.816664 avg=0.843084 VRAM=38.8GiB | 31.6% done | ETA(epoch): 116s + Train loss: 0.843143 (1272.3s) ETA: 7905min + Val loss: 0.877149 [t_0.0-0.2=1.0630 t_0.2-0.4=1.0233 t_0.4-0.6=0.8492 t_0.6-0.8=0.6894 t_0.8-1.0=0.6472] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 158/499 (32% done) --- + [Epoch 158][10/66] loss=0.886670 avg=0.871366 VRAM=39.0GiB | 31.6% done | ETA(epoch): 1075s + [Epoch 158][20/66] loss=0.781091 avg=0.851727 VRAM=38.8GiB | 31.7% done | ETA(epoch): 888s + [Epoch 158][30/66] loss=0.855128 avg=0.862240 VRAM=39.0GiB | 31.7% done | ETA(epoch): 694s + [Epoch 158][40/66] loss=0.772693 avg=0.857938 VRAM=38.8GiB | 31.7% done | ETA(epoch): 501s + [Epoch 158][50/66] loss=0.845239 avg=0.852335 VRAM=39.0GiB | 31.8% done | ETA(epoch): 308s + [Epoch 158][60/66] loss=0.848698 avg=0.847856 VRAM=38.8GiB | 31.8% done | ETA(epoch): 115s + Train loss: 0.850330 (1269.4s) ETA: 7881min + Val loss: 0.825440 [t_0.0-0.2=1.0603 t_0.2-0.4=0.9963 t_0.4-0.6=0.8326 t_0.6-0.8=0.7090 t_0.8-1.0=0.6175] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 159/499 (32% done) --- + [Epoch 159][10/66] loss=0.875712 avg=0.802747 VRAM=39.0GiB | 31.8% done | ETA(epoch): 1085s + [Epoch 159][20/66] loss=0.796735 avg=0.843299 VRAM=38.8GiB | 31.9% done | ETA(epoch): 886s + [Epoch 159][30/66] loss=1.019482 avg=0.869907 VRAM=39.0GiB | 31.9% done | ETA(epoch): 693s + [Epoch 159][40/66] loss=0.851209 avg=0.865021 VRAM=38.8GiB | 31.9% done | ETA(epoch): 500s + [Epoch 159][50/66] loss=1.019109 avg=0.867743 VRAM=39.0GiB | 32.0% done | ETA(epoch): 308s + [Epoch 159][60/66] loss=0.804183 avg=0.861258 VRAM=38.8GiB | 32.0% done | ETA(epoch): 115s + Train loss: 0.861212 (1268.4s) ETA: 7857min + Val loss: 0.858728 [t_0.0-0.2=1.0688 t_0.2-0.4=1.0028 t_0.4-0.6=0.8021 t_0.6-0.8=0.7100 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 160/499 (32% done) --- + [Epoch 160][10/66] loss=0.820757 avg=0.860027 VRAM=39.0GiB | 32.0% done | ETA(epoch): 1074s + [Epoch 160][20/66] loss=0.903262 avg=0.851317 VRAM=38.8GiB | 32.1% done | ETA(epoch): 883s + [Epoch 160][30/66] loss=0.862035 avg=0.853727 VRAM=39.0GiB | 32.1% done | ETA(epoch): 692s + [Epoch 160][40/66] loss=0.819784 avg=0.849271 VRAM=38.8GiB | 32.1% done | ETA(epoch): 499s + [Epoch 160][50/66] loss=0.841867 avg=0.847513 VRAM=39.0GiB | 32.2% done | ETA(epoch): 307s + [Epoch 160][60/66] loss=0.913840 avg=0.843245 VRAM=38.8GiB | 32.2% done | ETA(epoch): 115s + Train loss: 0.845587 (1268.5s) ETA: 7834min + Val loss: 0.843698 [t_0.0-0.2=1.0660 t_0.2-0.4=1.0292 t_0.4-0.6=0.8111 t_0.6-0.8=0.7093 t_0.8-1.0=0.6235] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 161/499 (32% done) --- + [Epoch 161][10/66] loss=0.787767 avg=0.811670 VRAM=39.0GiB | 32.2% done | ETA(epoch): 1076s + [Epoch 161][20/66] loss=0.734807 avg=0.836958 VRAM=38.8GiB | 32.3% done | ETA(epoch): 884s + [Epoch 161][30/66] loss=0.798653 avg=0.851808 VRAM=39.0GiB | 32.3% done | ETA(epoch): 692s + [Epoch 161][40/66] loss=0.908150 avg=0.844593 VRAM=38.8GiB | 32.3% done | ETA(epoch): 500s + [Epoch 161][50/66] loss=0.965235 avg=0.849889 VRAM=39.0GiB | 32.4% done | ETA(epoch): 308s + [Epoch 161][60/66] loss=0.809051 avg=0.843729 VRAM=38.8GiB | 32.4% done | ETA(epoch): 115s + Train loss: 0.840630 (1270.3s) ETA: 7810min + Val loss: 0.871054 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9924 t_0.4-0.6=0.8470 t_0.6-0.8=0.7348 t_0.8-1.0=0.6344] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 162/499 (32% done) --- + [Epoch 162][10/66] loss=0.825031 avg=0.864330 VRAM=39.0GiB | 32.4% done | ETA(epoch): 1076s + [Epoch 162][20/66] loss=0.871964 avg=0.849950 VRAM=38.8GiB | 32.5% done | ETA(epoch): 884s + [Epoch 162][30/66] loss=0.901309 avg=0.847335 VRAM=39.0GiB | 32.5% done | ETA(epoch): 691s + [Epoch 162][40/66] loss=0.828297 avg=0.851237 VRAM=38.8GiB | 32.5% done | ETA(epoch): 500s + [Epoch 162][50/66] loss=0.938394 avg=0.858684 VRAM=39.0GiB | 32.6% done | ETA(epoch): 307s + [Epoch 162][60/66] loss=0.795560 avg=0.858514 VRAM=38.8GiB | 32.6% done | ETA(epoch): 115s + Train loss: 0.855613 (1268.4s) ETA: 7786min + Val loss: 0.805565 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0083 t_0.4-0.6=0.8404 t_0.6-0.8=0.7178 t_0.8-1.0=0.6116] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0162 (BEST) + Deleted old checkpoint: checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 163/499 (33% done) --- + [Epoch 163][10/66] loss=0.683564 avg=0.834790 VRAM=39.0GiB | 32.6% done | ETA(epoch): 1075s + [Epoch 163][20/66] loss=0.867060 avg=0.870652 VRAM=38.8GiB | 32.7% done | ETA(epoch): 884s + [Epoch 163][30/66] loss=0.767124 avg=0.864834 VRAM=39.0GiB | 32.7% done | ETA(epoch): 691s + [Epoch 163][40/66] loss=0.867315 avg=0.863102 VRAM=38.8GiB | 32.7% done | ETA(epoch): 501s + [Epoch 163][50/66] loss=0.873857 avg=0.865663 VRAM=39.0GiB | 32.8% done | ETA(epoch): 308s + [Epoch 163][60/66] loss=1.004438 avg=0.867504 VRAM=38.8GiB | 32.8% done | ETA(epoch): 115s + Train loss: 0.863280 (1270.0s) ETA: 7762min + Val loss: 0.871441 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9634 t_0.4-0.6=0.8740 t_0.6-0.8=0.7121 t_0.8-1.0=0.6155] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 164/499 (33% done) --- + [Epoch 164][10/66] loss=0.943348 avg=0.919125 VRAM=39.0GiB | 32.8% done | ETA(epoch): 1077s + [Epoch 164][20/66] loss=0.803983 avg=0.878211 VRAM=38.8GiB | 32.9% done | ETA(epoch): 884s + [Epoch 164][30/66] loss=0.815765 avg=0.861440 VRAM=39.0GiB | 32.9% done | ETA(epoch): 692s + [Epoch 164][40/66] loss=0.871400 avg=0.866414 VRAM=38.8GiB | 32.9% done | ETA(epoch): 500s + [Epoch 164][50/66] loss=0.780240 avg=0.867038 VRAM=39.0GiB | 33.0% done | ETA(epoch): 308s + [Epoch 164][60/66] loss=0.732319 avg=0.859220 VRAM=38.8GiB | 33.0% done | ETA(epoch): 115s + Train loss: 0.856169 (1269.2s) ETA: 7738min + Val loss: 0.886499 [t_0.0-0.2=1.0774 t_0.2-0.4=0.9798 t_0.4-0.6=0.8447 t_0.6-0.8=0.7112 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 165/499 (33% done) --- + [Epoch 165][10/66] loss=0.835249 avg=0.860207 VRAM=39.0GiB | 33.0% done | ETA(epoch): 1077s + [Epoch 165][20/66] loss=0.982713 avg=0.869161 VRAM=38.8GiB | 33.1% done | ETA(epoch): 890s + [Epoch 165][30/66] loss=0.901620 avg=0.867624 VRAM=39.0GiB | 33.1% done | ETA(epoch): 695s + [Epoch 165][40/66] loss=0.921590 avg=0.868931 VRAM=38.8GiB | 33.1% done | ETA(epoch): 502s + [Epoch 165][50/66] loss=0.803137 avg=0.866831 VRAM=39.0GiB | 33.2% done | ETA(epoch): 309s + [Epoch 165][60/66] loss=0.859309 avg=0.865588 VRAM=38.8GiB | 33.2% done | ETA(epoch): 116s + Train loss: 0.867126 (1272.4s) ETA: 7715min + Val loss: 0.859728 [t_0.0-0.2=1.0707 t_0.2-0.4=1.0416 t_0.4-0.6=0.8352 t_0.6-0.8=0.6978 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0165 +[MEM @ epoch 165 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 166/499 (33% done) --- + [Epoch 166][10/66] loss=0.767566 avg=0.828931 VRAM=39.0GiB | 33.2% done | ETA(epoch): 1087s + [Epoch 166][20/66] loss=0.904547 avg=0.847124 VRAM=38.8GiB | 33.3% done | ETA(epoch): 889s + [Epoch 166][30/66] loss=0.762412 avg=0.855271 VRAM=39.0GiB | 33.3% done | ETA(epoch): 695s + [Epoch 166][40/66] loss=0.836633 avg=0.848615 VRAM=38.8GiB | 33.3% done | ETA(epoch): 501s + [Epoch 166][50/66] loss=0.832738 avg=0.842615 VRAM=39.0GiB | 33.4% done | ETA(epoch): 308s + [Epoch 166][60/66] loss=0.969806 avg=0.849201 VRAM=38.8GiB | 33.4% done | ETA(epoch): 116s + Train loss: 0.847850 (1271.5s) ETA: 7691min + Val loss: 0.819631 [t_0.0-0.2=1.0930 t_0.2-0.4=0.9668 t_0.4-0.6=0.8323 t_0.6-0.8=0.7019 t_0.8-1.0=0.6420] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 167/499 (33% done) --- + [Epoch 167][10/66] loss=0.830657 avg=0.861318 VRAM=39.0GiB | 33.4% done | ETA(epoch): 1076s + [Epoch 167][20/66] loss=0.729231 avg=0.860838 VRAM=38.8GiB | 33.5% done | ETA(epoch): 884s + [Epoch 167][30/66] loss=0.862909 avg=0.846912 VRAM=39.0GiB | 33.5% done | ETA(epoch): 692s + [Epoch 167][40/66] loss=0.919214 avg=0.839179 VRAM=38.8GiB | 33.5% done | ETA(epoch): 500s + [Epoch 167][50/66] loss=0.843338 avg=0.839133 VRAM=39.0GiB | 33.6% done | ETA(epoch): 308s + [Epoch 167][60/66] loss=0.845708 avg=0.836752 VRAM=38.8GiB | 33.6% done | ETA(epoch): 115s + Train loss: 0.840165 (1273.3s) ETA: 7667min + Val loss: 0.830066 [t_0.0-0.2=1.0645 t_0.2-0.4=1.0300 t_0.4-0.6=0.8323 t_0.6-0.8=0.7027 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 168/499 (34% done) --- + [Epoch 168][10/66] loss=0.884269 avg=0.868077 VRAM=39.0GiB | 33.6% done | ETA(epoch): 1077s + [Epoch 168][20/66] loss=0.884485 avg=0.851659 VRAM=38.8GiB | 33.7% done | ETA(epoch): 884s + [Epoch 168][30/66] loss=0.956552 avg=0.870023 VRAM=39.0GiB | 33.7% done | ETA(epoch): 692s + [Epoch 168][40/66] loss=0.802923 avg=0.861100 VRAM=38.8GiB | 33.7% done | ETA(epoch): 500s + [Epoch 168][50/66] loss=0.864539 avg=0.858973 VRAM=39.0GiB | 33.8% done | ETA(epoch): 308s + [Epoch 168][60/66] loss=0.688593 avg=0.856090 VRAM=38.8GiB | 33.8% done | ETA(epoch): 116s + Train loss: 0.851228 (1271.5s) ETA: 7643min + Val loss: 0.818639 [t_0.0-0.2=1.0601 t_0.2-0.4=0.9996 t_0.4-0.6=0.8466 t_0.6-0.8=0.7241 t_0.8-1.0=0.6413] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 169/499 (34% done) --- + [Epoch 169][10/66] loss=0.785288 avg=0.829481 VRAM=39.0GiB | 33.8% done | ETA(epoch): 1076s + [Epoch 169][20/66] loss=0.981449 avg=0.848623 VRAM=38.8GiB | 33.9% done | ETA(epoch): 884s + [Epoch 169][30/66] loss=0.895812 avg=0.858863 VRAM=39.0GiB | 33.9% done | ETA(epoch): 692s + [Epoch 169][40/66] loss=0.761983 avg=0.853024 VRAM=38.8GiB | 33.9% done | ETA(epoch): 500s + [Epoch 169][50/66] loss=0.822774 avg=0.861402 VRAM=39.0GiB | 34.0% done | ETA(epoch): 307s + [Epoch 169][60/66] loss=0.853471 avg=0.864560 VRAM=38.8GiB | 34.0% done | ETA(epoch): 115s + Train loss: 0.862341 (1268.3s) ETA: 7620min + Val loss: 0.865478 [t_0.0-0.2=1.0764 t_0.2-0.4=1.0208 t_0.4-0.6=0.8337 t_0.6-0.8=0.6731 t_0.8-1.0=0.6393] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 170/499 (34% done) --- + [Epoch 170][10/66] loss=0.902630 avg=0.822128 VRAM=39.0GiB | 34.0% done | ETA(epoch): 1077s + [Epoch 170][20/66] loss=0.925888 avg=0.838632 VRAM=38.8GiB | 34.1% done | ETA(epoch): 885s + [Epoch 170][30/66] loss=0.913030 avg=0.841386 VRAM=39.0GiB | 34.1% done | ETA(epoch): 693s + [Epoch 170][40/66] loss=0.960768 avg=0.849441 VRAM=38.8GiB | 34.1% done | ETA(epoch): 501s + [Epoch 170][50/66] loss=0.871844 avg=0.845765 VRAM=39.0GiB | 34.2% done | ETA(epoch): 308s + [Epoch 170][60/66] loss=0.916560 avg=0.849619 VRAM=38.8GiB | 34.2% done | ETA(epoch): 115s + Train loss: 0.848564 (1270.2s) ETA: 7596min + Val loss: 0.842578 [t_0.0-0.2=1.0706 t_0.2-0.4=0.9996 t_0.4-0.6=0.8622 t_0.6-0.8=0.6881 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 171/499 (34% done) --- + [Epoch 171][10/66] loss=0.823425 avg=0.846609 VRAM=39.0GiB | 34.2% done | ETA(epoch): 1076s + [Epoch 171][20/66] loss=0.760628 avg=0.844013 VRAM=38.8GiB | 34.3% done | ETA(epoch): 884s + [Epoch 171][30/66] loss=0.788438 avg=0.847093 VRAM=39.0GiB | 34.3% done | ETA(epoch): 693s + [Epoch 171][40/66] loss=0.830153 avg=0.850119 VRAM=38.8GiB | 34.3% done | ETA(epoch): 500s + [Epoch 171][50/66] loss=0.834098 avg=0.853778 VRAM=39.0GiB | 34.4% done | ETA(epoch): 308s + [Epoch 171][60/66] loss=0.745118 avg=0.854804 VRAM=38.8GiB | 34.4% done | ETA(epoch): 115s + Train loss: 0.852438 (1269.6s) ETA: 7572min + Val loss: 0.819408 [t_0.0-0.2=1.0720 t_0.2-0.4=0.9970 t_0.4-0.6=0.8500 t_0.6-0.8=0.7047 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 172/499 (34% done) --- + [Epoch 172][10/66] loss=0.654709 avg=0.859137 VRAM=39.0GiB | 34.4% done | ETA(epoch): 1078s + [Epoch 172][20/66] loss=0.808794 avg=0.852875 VRAM=38.8GiB | 34.5% done | ETA(epoch): 885s + [Epoch 172][30/66] loss=0.793860 avg=0.847965 VRAM=39.0GiB | 34.5% done | ETA(epoch): 692s + [Epoch 172][40/66] loss=0.865810 avg=0.845870 VRAM=38.8GiB | 34.5% done | ETA(epoch): 500s + [Epoch 172][50/66] loss=0.875940 avg=0.845940 VRAM=39.0GiB | 34.6% done | ETA(epoch): 308s + [Epoch 172][60/66] loss=0.875827 avg=0.852470 VRAM=38.8GiB | 34.6% done | ETA(epoch): 115s + Train loss: 0.853859 (1269.3s) ETA: 7548min + Val loss: 0.832808 [t_0.0-0.2=1.0497 t_0.2-0.4=1.0133 t_0.4-0.6=0.8527 t_0.6-0.8=0.7218 t_0.8-1.0=0.6461] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 173/499 (35% done) --- + [Epoch 173][10/66] loss=0.941034 avg=0.889270 VRAM=39.0GiB | 34.6% done | ETA(epoch): 1077s + [Epoch 173][20/66] loss=0.818277 avg=0.877378 VRAM=38.8GiB | 34.7% done | ETA(epoch): 885s + [Epoch 173][30/66] loss=0.907359 avg=0.865237 VRAM=39.0GiB | 34.7% done | ETA(epoch): 692s + [Epoch 173][40/66] loss=0.823930 avg=0.856992 VRAM=38.8GiB | 34.7% done | ETA(epoch): 500s + [Epoch 173][50/66] loss=0.739816 avg=0.852538 VRAM=39.0GiB | 34.8% done | ETA(epoch): 308s + [Epoch 173][60/66] loss=0.912637 avg=0.847196 VRAM=38.8GiB | 34.8% done | ETA(epoch): 115s + Train loss: 0.847419 (1268.7s) ETA: 7525min + Val loss: 0.867848 [t_0.0-0.2=1.0898 t_0.2-0.4=0.9904 t_0.4-0.6=0.8367 t_0.6-0.8=0.6898 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 174/499 (35% done) --- + [Epoch 174][10/66] loss=0.786178 avg=0.795197 VRAM=39.0GiB | 34.8% done | ETA(epoch): 1075s + [Epoch 174][20/66] loss=0.923029 avg=0.854261 VRAM=38.8GiB | 34.9% done | ETA(epoch): 883s + [Epoch 174][30/66] loss=0.889874 avg=0.854958 VRAM=39.0GiB | 34.9% done | ETA(epoch): 691s + [Epoch 174][40/66] loss=0.866419 avg=0.857476 VRAM=38.8GiB | 34.9% done | ETA(epoch): 500s + [Epoch 174][50/66] loss=0.787549 avg=0.858859 VRAM=39.0GiB | 35.0% done | ETA(epoch): 307s + [Epoch 174][60/66] loss=0.804347 avg=0.860172 VRAM=38.8GiB | 35.0% done | ETA(epoch): 115s + Train loss: 0.855491 (1268.2s) ETA: 7501min + Val loss: 0.858872 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9850 t_0.4-0.6=0.8523 t_0.6-0.8=0.7162 t_0.8-1.0=0.6241] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 175/499 (35% done) --- + [Epoch 175][10/66] loss=0.823657 avg=0.832512 VRAM=39.0GiB | 35.0% done | ETA(epoch): 1076s + [Epoch 175][20/66] loss=0.897491 avg=0.857251 VRAM=38.8GiB | 35.1% done | ETA(epoch): 884s + [Epoch 175][30/66] loss=0.936524 avg=0.862543 VRAM=39.0GiB | 35.1% done | ETA(epoch): 692s + [Epoch 175][40/66] loss=0.853685 avg=0.854428 VRAM=38.8GiB | 35.1% done | ETA(epoch): 500s + [Epoch 175][50/66] loss=0.922435 avg=0.857440 VRAM=39.0GiB | 35.2% done | ETA(epoch): 307s + [Epoch 175][60/66] loss=0.830533 avg=0.858595 VRAM=38.8GiB | 35.2% done | ETA(epoch): 115s + Train loss: 0.855689 (1270.3s) ETA: 7477min + Val loss: 0.851020 [t_0.0-0.2=1.0753 t_0.2-0.4=0.9574 t_0.4-0.6=0.8563 t_0.6-0.8=0.6933 t_0.8-1.0=0.6311] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 176/499 (35% done) --- + [Epoch 176][10/66] loss=0.768222 avg=0.863485 VRAM=39.0GiB | 35.2% done | ETA(epoch): 1075s + [Epoch 176][20/66] loss=0.826119 avg=0.864448 VRAM=38.8GiB | 35.3% done | ETA(epoch): 884s + [Epoch 176][30/66] loss=0.779310 avg=0.857873 VRAM=39.0GiB | 35.3% done | ETA(epoch): 692s + [Epoch 176][40/66] loss=0.837469 avg=0.853609 VRAM=38.8GiB | 35.3% done | ETA(epoch): 500s + [Epoch 176][50/66] loss=0.764197 avg=0.851818 VRAM=39.0GiB | 35.4% done | ETA(epoch): 308s + [Epoch 176][60/66] loss=0.786909 avg=0.845113 VRAM=38.8GiB | 35.4% done | ETA(epoch): 116s + Train loss: 0.843358 (1270.7s) ETA: 7453min + Val loss: 0.868543 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9972 t_0.4-0.6=0.8094 t_0.6-0.8=0.7275 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 177/499 (35% done) --- + [Epoch 177][10/66] loss=0.799252 avg=0.811367 VRAM=39.0GiB | 35.4% done | ETA(epoch): 1075s + [Epoch 177][20/66] loss=0.992142 avg=0.839761 VRAM=38.8GiB | 35.5% done | ETA(epoch): 883s + [Epoch 177][30/66] loss=0.941617 avg=0.851512 VRAM=39.0GiB | 35.5% done | ETA(epoch): 691s + [Epoch 177][40/66] loss=0.802229 avg=0.847656 VRAM=38.8GiB | 35.5% done | ETA(epoch): 499s + [Epoch 177][50/66] loss=0.850708 avg=0.845236 VRAM=39.0GiB | 35.6% done | ETA(epoch): 308s + [Epoch 177][60/66] loss=0.966938 avg=0.848972 VRAM=38.8GiB | 35.6% done | ETA(epoch): 115s + Train loss: 0.848676 (1269.7s) ETA: 7430min + Val loss: 0.835689 [t_0.0-0.2=1.0608 t_0.2-0.4=1.0007 t_0.4-0.6=0.8301 t_0.6-0.8=0.7056 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 178/499 (36% done) --- + [Epoch 178][10/66] loss=0.828885 avg=0.845412 VRAM=39.0GiB | 35.6% done | ETA(epoch): 1076s + [Epoch 178][20/66] loss=0.838593 avg=0.847782 VRAM=38.8GiB | 35.7% done | ETA(epoch): 884s + [Epoch 178][30/66] loss=0.891331 avg=0.843060 VRAM=39.0GiB | 35.7% done | ETA(epoch): 692s + [Epoch 178][40/66] loss=0.908854 avg=0.851114 VRAM=38.8GiB | 35.7% done | ETA(epoch): 505s + [Epoch 178][50/66] loss=0.815574 avg=0.847783 VRAM=39.0GiB | 35.8% done | ETA(epoch): 310s + [Epoch 178][60/66] loss=0.860053 avg=0.848523 VRAM=38.8GiB | 35.8% done | ETA(epoch): 116s + Train loss: 0.848568 (1276.3s) ETA: 7406min + Val loss: 0.844524 [t_0.0-0.2=1.0760 t_0.2-0.4=1.0323 t_0.4-0.6=0.8372 t_0.6-0.8=0.7232 t_0.8-1.0=0.6213] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 179/499 (36% done) --- + [Epoch 179][10/66] loss=0.810084 avg=0.839532 VRAM=39.0GiB | 35.8% done | ETA(epoch): 1075s + [Epoch 179][20/66] loss=0.883819 avg=0.845878 VRAM=38.8GiB | 35.9% done | ETA(epoch): 883s + [Epoch 179][30/66] loss=0.834348 avg=0.853479 VRAM=39.0GiB | 35.9% done | ETA(epoch): 691s + [Epoch 179][40/66] loss=0.856148 avg=0.854946 VRAM=38.8GiB | 35.9% done | ETA(epoch): 499s + [Epoch 179][50/66] loss=0.989195 avg=0.855540 VRAM=39.0GiB | 36.0% done | ETA(epoch): 307s + [Epoch 179][60/66] loss=0.722698 avg=0.847791 VRAM=38.8GiB | 36.0% done | ETA(epoch): 115s + Train loss: 0.851186 (1268.3s) ETA: 7383min + Val loss: 0.818509 [t_0.0-0.2=1.0675 t_0.2-0.4=0.9961 t_0.4-0.6=0.8313 t_0.6-0.8=0.7135 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 180/499 (36% done) --- + [Epoch 180][10/66] loss=0.918855 avg=0.870921 VRAM=39.0GiB | 36.0% done | ETA(epoch): 1076s + [Epoch 180][20/66] loss=0.833733 avg=0.862324 VRAM=38.8GiB | 36.1% done | ETA(epoch): 885s + [Epoch 180][30/66] loss=0.821700 avg=0.863692 VRAM=39.0GiB | 36.1% done | ETA(epoch): 692s + [Epoch 180][40/66] loss=0.882243 avg=0.848644 VRAM=38.8GiB | 36.1% done | ETA(epoch): 500s + [Epoch 180][50/66] loss=0.791921 avg=0.854804 VRAM=39.0GiB | 36.2% done | ETA(epoch): 308s + [Epoch 180][60/66] loss=0.778879 avg=0.850983 VRAM=38.8GiB | 36.2% done | ETA(epoch): 115s + Train loss: 0.847160 (1269.1s) ETA: 7359min + Val loss: 0.828829 [t_0.0-0.2=1.0607 t_0.2-0.4=0.9782 t_0.4-0.6=0.8126 t_0.6-0.8=0.6723 t_0.8-1.0=0.6299] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 181/499 (36% done) --- + [Epoch 181][10/66] loss=0.811618 avg=0.836261 VRAM=39.0GiB | 36.2% done | ETA(epoch): 1079s + [Epoch 181][20/66] loss=0.702888 avg=0.814667 VRAM=38.8GiB | 36.3% done | ETA(epoch): 885s + [Epoch 181][30/66] loss=0.886011 avg=0.820017 VRAM=39.0GiB | 36.3% done | ETA(epoch): 692s + [Epoch 181][40/66] loss=0.806698 avg=0.827713 VRAM=38.8GiB | 36.3% done | ETA(epoch): 500s + [Epoch 181][50/66] loss=0.994534 avg=0.843235 VRAM=39.0GiB | 36.4% done | ETA(epoch): 308s + [Epoch 181][60/66] loss=0.758120 avg=0.841785 VRAM=38.8GiB | 36.4% done | ETA(epoch): 115s + Train loss: 0.839899 (1269.3s) ETA: 7335min + Val loss: 0.810073 [t_0.0-0.2=1.0747 t_0.2-0.4=1.0084 t_0.4-0.6=0.8194 t_0.6-0.8=0.7171 t_0.8-1.0=0.6304] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 182/499 (36% done) --- + [Epoch 182][10/66] loss=0.922521 avg=0.881089 VRAM=39.0GiB | 36.4% done | ETA(epoch): 1076s + [Epoch 182][20/66] loss=0.715261 avg=0.864951 VRAM=38.8GiB | 36.5% done | ETA(epoch): 884s + [Epoch 182][30/66] loss=0.806889 avg=0.833673 VRAM=39.0GiB | 36.5% done | ETA(epoch): 692s + [Epoch 182][40/66] loss=0.766536 avg=0.838789 VRAM=38.8GiB | 36.5% done | ETA(epoch): 500s + [Epoch 182][50/66] loss=0.775146 avg=0.832607 VRAM=39.0GiB | 36.6% done | ETA(epoch): 307s + [Epoch 182][60/66] loss=0.756486 avg=0.822727 VRAM=38.8GiB | 36.6% done | ETA(epoch): 115s + Train loss: 0.822736 (1286.3s) ETA: 7312min + Val loss: 0.845265 [t_0.0-0.2=1.0774 t_0.2-0.4=0.9877 t_0.4-0.6=0.8591 t_0.6-0.8=0.7209 t_0.8-1.0=0.6215] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 183/499 (37% done) --- + [Epoch 183][10/66] loss=0.818104 avg=0.874000 VRAM=39.0GiB | 36.6% done | ETA(epoch): 1075s + [Epoch 183][20/66] loss=0.892491 avg=0.857502 VRAM=38.8GiB | 36.7% done | ETA(epoch): 884s + [Epoch 183][30/66] loss=0.926232 avg=0.860769 VRAM=39.0GiB | 36.7% done | ETA(epoch): 692s + [Epoch 183][40/66] loss=0.918208 avg=0.863711 VRAM=38.8GiB | 36.7% done | ETA(epoch): 500s + [Epoch 183][50/66] loss=0.885428 avg=0.855854 VRAM=39.0GiB | 36.8% done | ETA(epoch): 308s + [Epoch 183][60/66] loss=0.858131 avg=0.851904 VRAM=38.8GiB | 36.8% done | ETA(epoch): 116s + Train loss: 0.856391 (1272.5s) ETA: 7289min + Val loss: 0.817464 [t_0.0-0.2=1.0292 t_0.2-0.4=1.0115 t_0.4-0.6=0.8579 t_0.6-0.8=0.7056 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 184/499 (37% done) --- + [Epoch 184][10/66] loss=0.853351 avg=0.837900 VRAM=39.0GiB | 36.8% done | ETA(epoch): 1077s + [Epoch 184][20/66] loss=0.821292 avg=0.836836 VRAM=38.8GiB | 36.9% done | ETA(epoch): 884s + [Epoch 184][30/66] loss=0.920047 avg=0.840669 VRAM=39.0GiB | 36.9% done | ETA(epoch): 692s + [Epoch 184][40/66] loss=0.890025 avg=0.839791 VRAM=38.8GiB | 36.9% done | ETA(epoch): 500s + [Epoch 184][50/66] loss=0.815083 avg=0.843914 VRAM=39.0GiB | 37.0% done | ETA(epoch): 308s + [Epoch 184][60/66] loss=0.902734 avg=0.853245 VRAM=38.8GiB | 37.0% done | ETA(epoch): 115s + Train loss: 0.854620 (1268.6s) ETA: 7265min + Val loss: 0.851970 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0126 t_0.4-0.6=0.8147 t_0.6-0.8=0.7221 t_0.8-1.0=0.6234] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 185/499 (37% done) --- + [Epoch 185][10/66] loss=0.666568 avg=0.840407 VRAM=39.0GiB | 37.0% done | ETA(epoch): 1074s + [Epoch 185][20/66] loss=0.731730 avg=0.842259 VRAM=38.8GiB | 37.1% done | ETA(epoch): 885s + [Epoch 185][30/66] loss=0.736242 avg=0.839962 VRAM=39.0GiB | 37.1% done | ETA(epoch): 692s + [Epoch 185][40/66] loss=0.895081 avg=0.831321 VRAM=38.8GiB | 37.1% done | ETA(epoch): 500s + [Epoch 185][50/66] loss=0.837639 avg=0.833526 VRAM=39.0GiB | 37.2% done | ETA(epoch): 308s + [Epoch 185][60/66] loss=0.882776 avg=0.842970 VRAM=38.8GiB | 37.2% done | ETA(epoch): 115s + Train loss: 0.846934 (1268.6s) ETA: 7241min + Val loss: 0.822665 [t_0.0-0.2=1.0823 t_0.2-0.4=0.9857 t_0.4-0.6=0.8435 t_0.6-0.8=0.7002 t_0.8-1.0=0.6298] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 186/499 (37% done) --- + [Epoch 186][10/66] loss=0.916444 avg=0.854683 VRAM=39.0GiB | 37.2% done | ETA(epoch): 1078s + [Epoch 186][20/66] loss=0.771816 avg=0.834493 VRAM=38.8GiB | 37.3% done | ETA(epoch): 885s + [Epoch 186][30/66] loss=0.850320 avg=0.838636 VRAM=39.0GiB | 37.3% done | ETA(epoch): 695s + [Epoch 186][40/66] loss=0.835667 avg=0.843155 VRAM=38.8GiB | 37.3% done | ETA(epoch): 502s + [Epoch 186][50/66] loss=0.917323 avg=0.845721 VRAM=39.0GiB | 37.4% done | ETA(epoch): 308s + [Epoch 186][60/66] loss=0.712437 avg=0.843915 VRAM=38.8GiB | 37.4% done | ETA(epoch): 116s + Train loss: 0.844686 (1271.1s) ETA: 7218min + Val loss: 0.859124 [t_0.0-0.2=1.0662 t_0.2-0.4=0.9810 t_0.4-0.6=0.8154 t_0.6-0.8=0.6992 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 187/499 (37% done) --- + [Epoch 187][10/66] loss=0.842442 avg=0.836500 VRAM=39.0GiB | 37.4% done | ETA(epoch): 1076s + [Epoch 187][20/66] loss=0.861052 avg=0.841568 VRAM=38.8GiB | 37.5% done | ETA(epoch): 884s + [Epoch 187][30/66] loss=0.786211 avg=0.828098 VRAM=39.0GiB | 37.5% done | ETA(epoch): 692s + [Epoch 187][40/66] loss=0.845443 avg=0.830312 VRAM=38.8GiB | 37.5% done | ETA(epoch): 500s + [Epoch 187][50/66] loss=0.947152 avg=0.837877 VRAM=39.0GiB | 37.6% done | ETA(epoch): 307s + [Epoch 187][60/66] loss=0.733813 avg=0.833451 VRAM=38.8GiB | 37.6% done | ETA(epoch): 115s + Train loss: 0.829431 (1268.5s) ETA: 7194min + Val loss: 0.843496 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9839 t_0.4-0.6=0.8512 t_0.6-0.8=0.6767 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 188/499 (38% done) --- + [Epoch 188][10/66] loss=0.977309 avg=0.881047 VRAM=39.0GiB | 37.6% done | ETA(epoch): 1075s + [Epoch 188][20/66] loss=0.928108 avg=0.863560 VRAM=38.8GiB | 37.7% done | ETA(epoch): 883s + [Epoch 188][30/66] loss=0.886118 avg=0.852782 VRAM=39.0GiB | 37.7% done | ETA(epoch): 691s + [Epoch 188][40/66] loss=0.894279 avg=0.856570 VRAM=38.8GiB | 37.7% done | ETA(epoch): 499s + [Epoch 188][50/66] loss=0.969330 avg=0.856296 VRAM=39.0GiB | 37.8% done | ETA(epoch): 307s + [Epoch 188][60/66] loss=0.781460 avg=0.853198 VRAM=38.8GiB | 37.8% done | ETA(epoch): 115s + Train loss: 0.848352 (1268.2s) ETA: 7171min + Val loss: 0.805406 [t_0.0-0.2=1.0683 t_0.2-0.4=0.9473 t_0.4-0.6=0.8394 t_0.6-0.8=0.7007 t_0.8-1.0=0.6252] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0188 (BEST) + Deleted old checkpoint: checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 189/499 (38% done) --- + [Epoch 189][10/66] loss=0.864910 avg=0.849139 VRAM=39.0GiB | 37.8% done | ETA(epoch): 1075s + [Epoch 189][20/66] loss=0.878867 avg=0.848915 VRAM=38.8GiB | 37.9% done | ETA(epoch): 884s + [Epoch 189][30/66] loss=0.787418 avg=0.840587 VRAM=39.0GiB | 37.9% done | ETA(epoch): 692s + [Epoch 189][40/66] loss=0.855426 avg=0.837976 VRAM=38.8GiB | 37.9% done | ETA(epoch): 500s + [Epoch 189][50/66] loss=0.753021 avg=0.837618 VRAM=39.0GiB | 38.0% done | ETA(epoch): 308s + [Epoch 189][60/66] loss=0.810959 avg=0.836901 VRAM=38.8GiB | 38.0% done | ETA(epoch): 115s + Train loss: 0.836084 (1269.2s) ETA: 7147min + Val loss: 0.884837 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9872 t_0.4-0.6=0.8612 t_0.6-0.8=0.6924 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 190/499 (38% done) --- + [Epoch 190][10/66] loss=0.782788 avg=0.840810 VRAM=39.0GiB | 38.0% done | ETA(epoch): 1075s + [Epoch 190][20/66] loss=0.896518 avg=0.850262 VRAM=38.8GiB | 38.1% done | ETA(epoch): 883s + [Epoch 190][30/66] loss=0.843134 avg=0.852873 VRAM=39.0GiB | 38.1% done | ETA(epoch): 692s + [Epoch 190][40/66] loss=0.763332 avg=0.851478 VRAM=38.8GiB | 38.1% done | ETA(epoch): 500s + [Epoch 190][50/66] loss=0.994692 avg=0.855693 VRAM=39.0GiB | 38.2% done | ETA(epoch): 308s + [Epoch 190][60/66] loss=0.987656 avg=0.853828 VRAM=38.8GiB | 38.2% done | ETA(epoch): 116s + Train loss: 0.855561 (1270.7s) ETA: 7123min + Val loss: 0.836215 [t_0.0-0.2=1.0805 t_0.2-0.4=0.9788 t_0.4-0.6=0.8239 t_0.6-0.8=0.7164 t_0.8-1.0=0.6224] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 191/499 (38% done) --- + [Epoch 191][10/66] loss=0.865930 avg=0.866546 VRAM=39.0GiB | 38.2% done | ETA(epoch): 1074s + [Epoch 191][20/66] loss=0.927099 avg=0.860972 VRAM=38.8GiB | 38.3% done | ETA(epoch): 883s + [Epoch 191][30/66] loss=0.789475 avg=0.864764 VRAM=39.0GiB | 38.3% done | ETA(epoch): 692s + [Epoch 191][40/66] loss=0.849377 avg=0.858486 VRAM=38.8GiB | 38.3% done | ETA(epoch): 500s + [Epoch 191][50/66] loss=0.827272 avg=0.858127 VRAM=39.0GiB | 38.4% done | ETA(epoch): 307s + [Epoch 191][60/66] loss=0.789265 avg=0.847460 VRAM=38.8GiB | 38.4% done | ETA(epoch): 115s + Train loss: 0.844815 (1268.3s) ETA: 7100min + Val loss: 0.838562 [t_0.0-0.2=1.0738 t_0.2-0.4=1.0002 t_0.4-0.6=0.8176 t_0.6-0.8=0.7142 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0191 +[MEM @ epoch 191 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 192/499 (38% done) --- + [Epoch 192][10/66] loss=0.777885 avg=0.855372 VRAM=39.0GiB | 38.4% done | ETA(epoch): 1079s + [Epoch 192][20/66] loss=0.856058 avg=0.861999 VRAM=38.8GiB | 38.5% done | ETA(epoch): 888s + [Epoch 192][30/66] loss=0.892754 avg=0.847929 VRAM=39.0GiB | 38.5% done | ETA(epoch): 695s + [Epoch 192][40/66] loss=0.777278 avg=0.850519 VRAM=38.8GiB | 38.5% done | ETA(epoch): 502s + [Epoch 192][50/66] loss=0.877074 avg=0.851875 VRAM=39.0GiB | 38.6% done | ETA(epoch): 309s + [Epoch 192][60/66] loss=0.807752 avg=0.848853 VRAM=38.8GiB | 38.6% done | ETA(epoch): 116s + Train loss: 0.846537 (1273.9s) ETA: 7076min + Val loss: 0.831045 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9956 t_0.4-0.6=0.8370 t_0.6-0.8=0.7357 t_0.8-1.0=0.6353] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 193/499 (39% done) --- + [Epoch 193][10/66] loss=0.832400 avg=0.843345 VRAM=39.0GiB | 38.6% done | ETA(epoch): 1074s + [Epoch 193][20/66] loss=0.693555 avg=0.848137 VRAM=38.8GiB | 38.7% done | ETA(epoch): 883s + [Epoch 193][30/66] loss=0.802384 avg=0.849678 VRAM=39.0GiB | 38.7% done | ETA(epoch): 691s + [Epoch 193][40/66] loss=0.847911 avg=0.844983 VRAM=38.8GiB | 38.7% done | ETA(epoch): 504s + [Epoch 193][50/66] loss=0.873887 avg=0.846331 VRAM=39.0GiB | 38.8% done | ETA(epoch): 309s + [Epoch 193][60/66] loss=0.897982 avg=0.846431 VRAM=38.8GiB | 38.8% done | ETA(epoch): 116s + Train loss: 0.843636 (1274.4s) ETA: 7053min + Val loss: 0.871973 [t_0.0-0.2=1.0653 t_0.2-0.4=0.9840 t_0.4-0.6=0.8553 t_0.6-0.8=0.7099 t_0.8-1.0=0.6316] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 194/499 (39% done) --- + [Epoch 194][10/66] loss=0.726037 avg=0.859180 VRAM=39.0GiB | 38.8% done | ETA(epoch): 1075s + [Epoch 194][20/66] loss=0.875758 avg=0.872762 VRAM=38.8GiB | 38.9% done | ETA(epoch): 884s + [Epoch 194][30/66] loss=0.804173 avg=0.863514 VRAM=39.0GiB | 38.9% done | ETA(epoch): 692s + [Epoch 194][40/66] loss=0.744479 avg=0.861595 VRAM=38.8GiB | 38.9% done | ETA(epoch): 500s + [Epoch 194][50/66] loss=0.805191 avg=0.857451 VRAM=39.0GiB | 39.0% done | ETA(epoch): 307s + [Epoch 194][60/66] loss=0.842891 avg=0.857823 VRAM=38.8GiB | 39.0% done | ETA(epoch): 115s + Train loss: 0.853907 (1268.7s) ETA: 7029min + Val loss: 0.822062 [t_0.0-0.2=1.0451 t_0.2-0.4=0.9640 t_0.4-0.6=0.8507 t_0.6-0.8=0.6997 t_0.8-1.0=0.6042] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 195/499 (39% done) --- + [Epoch 195][10/66] loss=0.790558 avg=0.873794 VRAM=39.0GiB | 39.0% done | ETA(epoch): 1077s + [Epoch 195][20/66] loss=0.755216 avg=0.851742 VRAM=38.8GiB | 39.1% done | ETA(epoch): 885s + [Epoch 195][30/66] loss=0.895199 avg=0.858055 VRAM=39.0GiB | 39.1% done | ETA(epoch): 692s + [Epoch 195][40/66] loss=0.859901 avg=0.848336 VRAM=38.8GiB | 39.1% done | ETA(epoch): 500s + [Epoch 195][50/66] loss=0.940118 avg=0.851866 VRAM=39.0GiB | 39.2% done | ETA(epoch): 308s + [Epoch 195][60/66] loss=0.857839 avg=0.849205 VRAM=38.8GiB | 39.2% done | ETA(epoch): 115s + Train loss: 0.846720 (1269.2s) ETA: 7006min + Val loss: 0.813569 [t_0.0-0.2=1.0742 t_0.2-0.4=0.9754 t_0.4-0.6=0.8270 t_0.6-0.8=0.6792 t_0.8-1.0=0.6290] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 196/499 (39% done) --- + [Epoch 196][10/66] loss=0.767733 avg=0.826820 VRAM=39.0GiB | 39.2% done | ETA(epoch): 1098s + [Epoch 196][20/66] loss=0.729027 avg=0.823357 VRAM=38.8GiB | 39.3% done | ETA(epoch): 893s + [Epoch 196][30/66] loss=0.887192 avg=0.839484 VRAM=39.0GiB | 39.3% done | ETA(epoch): 697s + [Epoch 196][40/66] loss=0.874019 avg=0.840507 VRAM=38.8GiB | 39.3% done | ETA(epoch): 502s + [Epoch 196][50/66] loss=0.794514 avg=0.837665 VRAM=39.0GiB | 39.4% done | ETA(epoch): 309s + [Epoch 196][60/66] loss=0.791755 avg=0.842836 VRAM=38.8GiB | 39.4% done | ETA(epoch): 116s + Train loss: 0.844772 (1273.1s) ETA: 6983min + Val loss: 0.836411 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9952 t_0.4-0.6=0.8560 t_0.6-0.8=0.7129 t_0.8-1.0=0.6163] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 197/499 (39% done) --- + [Epoch 197][10/66] loss=0.832124 avg=0.836035 VRAM=39.0GiB | 39.4% done | ETA(epoch): 1075s + [Epoch 197][20/66] loss=0.818877 avg=0.828581 VRAM=38.8GiB | 39.5% done | ETA(epoch): 883s + [Epoch 197][30/66] loss=0.805738 avg=0.845695 VRAM=39.0GiB | 39.5% done | ETA(epoch): 691s + [Epoch 197][40/66] loss=0.811608 avg=0.837131 VRAM=38.8GiB | 39.5% done | ETA(epoch): 499s + [Epoch 197][50/66] loss=0.843555 avg=0.835338 VRAM=39.0GiB | 39.6% done | ETA(epoch): 307s + [Epoch 197][60/66] loss=0.761149 avg=0.836872 VRAM=38.8GiB | 39.6% done | ETA(epoch): 115s + Train loss: 0.838656 (1270.0s) ETA: 6959min + Val loss: 0.841720 [t_0.0-0.2=1.0650 t_0.2-0.4=0.9792 t_0.4-0.6=0.8417 t_0.6-0.8=0.7191 t_0.8-1.0=0.6366] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 198/499 (40% done) --- + [Epoch 198][10/66] loss=0.929894 avg=0.871571 VRAM=39.0GiB | 39.6% done | ETA(epoch): 1077s + [Epoch 198][20/66] loss=0.863522 avg=0.860781 VRAM=38.8GiB | 39.7% done | ETA(epoch): 884s + [Epoch 198][30/66] loss=1.020064 avg=0.864329 VRAM=39.0GiB | 39.7% done | ETA(epoch): 692s + [Epoch 198][40/66] loss=0.800617 avg=0.849275 VRAM=38.8GiB | 39.7% done | ETA(epoch): 500s + [Epoch 198][50/66] loss=0.938746 avg=0.846788 VRAM=39.0GiB | 39.8% done | ETA(epoch): 307s + [Epoch 198][60/66] loss=0.896625 avg=0.851932 VRAM=38.8GiB | 39.8% done | ETA(epoch): 116s + Train loss: 0.856097 (1270.9s) ETA: 6936min + Val loss: 0.848358 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9896 t_0.4-0.6=0.8554 t_0.6-0.8=0.6993 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 199/499 (40% done) --- + [Epoch 199][10/66] loss=0.927766 avg=0.869454 VRAM=39.0GiB | 39.8% done | ETA(epoch): 1075s + [Epoch 199][20/66] loss=0.813360 avg=0.862446 VRAM=38.8GiB | 39.9% done | ETA(epoch): 883s + [Epoch 199][30/66] loss=0.726782 avg=0.854745 VRAM=39.0GiB | 39.9% done | ETA(epoch): 691s + [Epoch 199][40/66] loss=0.696552 avg=0.849380 VRAM=38.8GiB | 39.9% done | ETA(epoch): 499s + [Epoch 199][50/66] loss=0.793658 avg=0.847420 VRAM=39.0GiB | 40.0% done | ETA(epoch): 307s + [Epoch 199][60/66] loss=0.744273 avg=0.843163 VRAM=38.8GiB | 40.0% done | ETA(epoch): 115s + Train loss: 0.850294 (1267.5s) ETA: 6912min + Val loss: 0.856513 [t_0.0-0.2=1.0754 t_0.2-0.4=0.9835 t_0.4-0.6=0.8137 t_0.6-0.8=0.7346 t_0.8-1.0=0.6177] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 200/499 (40% done) --- + [MilestoneVis] train_0 step 13200 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 13200 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 13200 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 13200 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 13200 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 13200 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 13200 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 13200 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 13200 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 13200 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 200][10/66] loss=0.763872 avg=0.874404 VRAM=39.0GiB | 40.0% done | ETA(epoch): 6840s + [Epoch 200][20/66] loss=0.813658 avg=0.845685 VRAM=38.8GiB | 40.1% done | ETA(epoch): 3252s + [Epoch 200][30/66] loss=0.811206 avg=0.845194 VRAM=39.0GiB | 40.1% done | ETA(epoch): 1927s + [Epoch 200][40/66] loss=0.780860 avg=0.844734 VRAM=38.8GiB | 40.1% done | ETA(epoch): 1169s + [Epoch 200][50/66] loss=0.729399 avg=0.841708 VRAM=39.0GiB | 40.2% done | ETA(epoch): 637s + [Epoch 200][60/66] loss=0.928981 avg=0.838266 VRAM=38.8GiB | 40.2% done | ETA(epoch): 218s + Train loss: 0.842058 (2297.9s) ETA: 6914min + Val loss: 0.828492 [t_0.0-0.2=1.0667 t_0.2-0.4=1.0141 t_0.4-0.6=0.8611 t_0.6-0.8=0.7176 t_0.8-1.0=0.6317] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 201/499 (40% done) --- + [Epoch 201][10/66] loss=0.872833 avg=0.879523 VRAM=39.0GiB | 40.2% done | ETA(epoch): 1075s + [Epoch 201][20/66] loss=0.937075 avg=0.871403 VRAM=38.8GiB | 40.3% done | ETA(epoch): 884s + [Epoch 201][30/66] loss=0.770461 avg=0.862558 VRAM=39.0GiB | 40.3% done | ETA(epoch): 692s + [Epoch 201][40/66] loss=0.969322 avg=0.865565 VRAM=38.8GiB | 40.3% done | ETA(epoch): 504s + [Epoch 201][50/66] loss=0.752732 avg=0.861232 VRAM=39.0GiB | 40.4% done | ETA(epoch): 309s + [Epoch 201][60/66] loss=0.789427 avg=0.865034 VRAM=38.8GiB | 40.4% done | ETA(epoch): 116s + Train loss: 0.861364 (1274.5s) ETA: 6891min + Val loss: 0.806004 [t_0.0-0.2=1.0741 t_0.2-0.4=0.9766 t_0.4-0.6=0.8490 t_0.6-0.8=0.6950 t_0.8-1.0=0.6386] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 202/499 (40% done) --- + [Epoch 202][10/66] loss=0.826293 avg=0.865783 VRAM=39.0GiB | 40.4% done | ETA(epoch): 1076s + [Epoch 202][20/66] loss=0.825234 avg=0.868832 VRAM=38.8GiB | 40.5% done | ETA(epoch): 884s + [Epoch 202][30/66] loss=0.876319 avg=0.857413 VRAM=39.0GiB | 40.5% done | ETA(epoch): 695s + [Epoch 202][40/66] loss=0.789071 avg=0.858650 VRAM=38.8GiB | 40.5% done | ETA(epoch): 501s + [Epoch 202][50/66] loss=0.703320 avg=0.852938 VRAM=39.0GiB | 40.6% done | ETA(epoch): 308s + [Epoch 202][60/66] loss=0.786588 avg=0.848874 VRAM=38.8GiB | 40.6% done | ETA(epoch): 115s + Train loss: 0.852673 (1269.6s) ETA: 6867min + Val loss: 0.864275 [t_0.0-0.2=1.0583 t_0.2-0.4=1.0135 t_0.4-0.6=0.8320 t_0.6-0.8=0.7081 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 19.3/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 203/499 (41% done) --- + [Epoch 203][10/66] loss=0.850822 avg=0.836315 VRAM=39.0GiB | 40.6% done | ETA(epoch): 1075s + [Epoch 203][20/66] loss=0.902593 avg=0.835235 VRAM=38.8GiB | 40.7% done | ETA(epoch): 884s + [Epoch 203][30/66] loss=0.842593 avg=0.829268 VRAM=39.0GiB | 40.7% done | ETA(epoch): 694s + [Epoch 203][40/66] loss=0.798826 avg=0.832147 VRAM=38.8GiB | 40.7% done | ETA(epoch): 501s + [Epoch 203][50/66] loss=0.868944 avg=0.835020 VRAM=39.0GiB | 40.8% done | ETA(epoch): 308s + [Epoch 203][60/66] loss=1.029198 avg=0.839701 VRAM=38.8GiB | 40.8% done | ETA(epoch): 115s + Train loss: 0.842062 (1269.8s) ETA: 6843min + Val loss: 0.834246 [t_0.0-0.2=1.0627 t_0.2-0.4=0.9940 t_0.4-0.6=0.8523 t_0.6-0.8=0.7237 t_0.8-1.0=0.6280] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 204/499 (41% done) --- + [Epoch 204][10/66] loss=0.764135 avg=0.860167 VRAM=39.0GiB | 40.8% done | ETA(epoch): 1075s + [Epoch 204][20/66] loss=0.942941 avg=0.875091 VRAM=38.8GiB | 40.9% done | ETA(epoch): 887s + [Epoch 204][30/66] loss=0.857944 avg=0.865755 VRAM=39.0GiB | 40.9% done | ETA(epoch): 694s + [Epoch 204][40/66] loss=0.804540 avg=0.856989 VRAM=38.8GiB | 40.9% done | ETA(epoch): 501s + [Epoch 204][50/66] loss=0.978543 avg=0.861222 VRAM=39.0GiB | 41.0% done | ETA(epoch): 308s + [Epoch 204][60/66] loss=0.883260 avg=0.862236 VRAM=38.8GiB | 41.0% done | ETA(epoch): 115s + Train loss: 0.859883 (1269.3s) ETA: 6820min + Val loss: 0.866011 [t_0.0-0.2=1.0655 t_0.2-0.4=0.9885 t_0.4-0.6=0.8613 t_0.6-0.8=0.7059 t_0.8-1.0=0.6325] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 205/499 (41% done) --- + [Epoch 205][10/66] loss=0.901488 avg=0.844576 VRAM=39.0GiB | 41.0% done | ETA(epoch): 1075s + [Epoch 205][20/66] loss=0.887412 avg=0.832564 VRAM=38.8GiB | 41.1% done | ETA(epoch): 883s + [Epoch 205][30/66] loss=0.963020 avg=0.841283 VRAM=39.0GiB | 41.1% done | ETA(epoch): 692s + [Epoch 205][40/66] loss=0.752697 avg=0.835085 VRAM=38.8GiB | 41.1% done | ETA(epoch): 499s + [Epoch 205][50/66] loss=0.898020 avg=0.831128 VRAM=39.0GiB | 41.2% done | ETA(epoch): 307s + [Epoch 205][60/66] loss=0.913966 avg=0.840773 VRAM=38.8GiB | 41.2% done | ETA(epoch): 115s + Train loss: 0.838363 (1267.8s) ETA: 6796min + Val loss: 0.831803 [t_0.0-0.2=1.0857 t_0.2-0.4=0.9853 t_0.4-0.6=0.8366 t_0.6-0.8=0.7030 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 206/499 (41% done) --- + [Epoch 206][10/66] loss=0.946756 avg=0.824317 VRAM=39.0GiB | 41.2% done | ETA(epoch): 1075s + [Epoch 206][20/66] loss=0.930702 avg=0.829178 VRAM=38.8GiB | 41.3% done | ETA(epoch): 883s + [Epoch 206][30/66] loss=0.955828 avg=0.827222 VRAM=39.0GiB | 41.3% done | ETA(epoch): 691s + [Epoch 206][40/66] loss=0.748704 avg=0.828283 VRAM=38.8GiB | 41.3% done | ETA(epoch): 500s + [Epoch 206][50/66] loss=0.758826 avg=0.833700 VRAM=39.0GiB | 41.4% done | ETA(epoch): 307s + [Epoch 206][60/66] loss=0.838958 avg=0.834378 VRAM=38.8GiB | 41.4% done | ETA(epoch): 115s + Train loss: 0.834737 (1268.0s) ETA: 6772min + Val loss: 0.827199 [t_0.0-0.2=1.0632 t_0.2-0.4=1.0514 t_0.4-0.6=0.8317 t_0.6-0.8=0.7151 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 207/499 (41% done) --- + [Epoch 207][10/66] loss=0.845205 avg=0.855908 VRAM=39.0GiB | 41.4% done | ETA(epoch): 1075s + [Epoch 207][20/66] loss=0.834484 avg=0.866113 VRAM=38.8GiB | 41.5% done | ETA(epoch): 884s + [Epoch 207][30/66] loss=0.752339 avg=0.854173 VRAM=39.0GiB | 41.5% done | ETA(epoch): 692s + [Epoch 207][40/66] loss=0.830389 avg=0.851370 VRAM=38.8GiB | 41.5% done | ETA(epoch): 500s + [Epoch 207][50/66] loss=0.855402 avg=0.848254 VRAM=39.0GiB | 41.6% done | ETA(epoch): 308s + [Epoch 207][60/66] loss=1.000875 avg=0.849949 VRAM=38.8GiB | 41.6% done | ETA(epoch): 115s + Train loss: 0.849960 (1269.2s) ETA: 6749min + Val loss: 0.830051 [t_0.0-0.2=1.0833 t_0.2-0.4=1.0128 t_0.4-0.6=0.8319 t_0.6-0.8=0.7196 t_0.8-1.0=0.6228] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 208/499 (42% done) --- + [Epoch 208][10/66] loss=0.833659 avg=0.878907 VRAM=39.0GiB | 41.6% done | ETA(epoch): 1076s + [Epoch 208][20/66] loss=0.851534 avg=0.888213 VRAM=38.8GiB | 41.7% done | ETA(epoch): 885s + [Epoch 208][30/66] loss=0.917952 avg=0.887314 VRAM=39.0GiB | 41.7% done | ETA(epoch): 692s + [Epoch 208][40/66] loss=0.715053 avg=0.863831 VRAM=38.8GiB | 41.7% done | ETA(epoch): 500s + [Epoch 208][50/66] loss=0.824643 avg=0.859908 VRAM=39.0GiB | 41.8% done | ETA(epoch): 308s + [Epoch 208][60/66] loss=0.878554 avg=0.854552 VRAM=38.8GiB | 41.8% done | ETA(epoch): 115s + Train loss: 0.853775 (1269.5s) ETA: 6725min + Val loss: 0.849439 [t_0.0-0.2=1.0550 t_0.2-0.4=1.0111 t_0.4-0.6=0.8234 t_0.6-0.8=0.7268 t_0.8-1.0=0.6287] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 209/499 (42% done) --- + [Epoch 209][10/66] loss=0.928743 avg=0.852308 VRAM=39.0GiB | 41.8% done | ETA(epoch): 1077s + [Epoch 209][20/66] loss=0.980040 avg=0.882071 VRAM=38.8GiB | 41.9% done | ETA(epoch): 884s + [Epoch 209][30/66] loss=0.851163 avg=0.877086 VRAM=39.0GiB | 41.9% done | ETA(epoch): 693s + [Epoch 209][40/66] loss=0.898023 avg=0.862056 VRAM=38.8GiB | 41.9% done | ETA(epoch): 500s + [Epoch 209][50/66] loss=0.936335 avg=0.849386 VRAM=39.0GiB | 42.0% done | ETA(epoch): 308s + [Epoch 209][60/66] loss=0.781914 avg=0.847419 VRAM=38.8GiB | 42.0% done | ETA(epoch): 115s + Train loss: 0.845022 (1269.5s) ETA: 6701min + Val loss: 0.864104 [t_0.0-0.2=1.0728 t_0.2-0.4=1.0172 t_0.4-0.6=0.8461 t_0.6-0.8=0.7087 t_0.8-1.0=0.6285] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 210/499 (42% done) --- + [Epoch 210][10/66] loss=0.861422 avg=0.828721 VRAM=39.0GiB | 42.0% done | ETA(epoch): 1076s + [Epoch 210][20/66] loss=0.898670 avg=0.827783 VRAM=38.8GiB | 42.1% done | ETA(epoch): 885s + [Epoch 210][30/66] loss=0.881720 avg=0.820182 VRAM=39.0GiB | 42.1% done | ETA(epoch): 693s + [Epoch 210][40/66] loss=0.944499 avg=0.821746 VRAM=38.8GiB | 42.1% done | ETA(epoch): 500s + [Epoch 210][50/66] loss=0.850138 avg=0.825672 VRAM=39.0GiB | 42.2% done | ETA(epoch): 308s + [Epoch 210][60/66] loss=0.740058 avg=0.838412 VRAM=38.8GiB | 42.2% done | ETA(epoch): 115s + Train loss: 0.840212 (1270.0s) ETA: 6678min + Val loss: 0.834489 [t_0.0-0.2=1.0745 t_0.2-0.4=0.9974 t_0.4-0.6=0.8627 t_0.6-0.8=0.6974 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 211/499 (42% done) --- + [Epoch 211][10/66] loss=0.920758 avg=0.824424 VRAM=39.0GiB | 42.2% done | ETA(epoch): 1076s + [Epoch 211][20/66] loss=0.846325 avg=0.838382 VRAM=38.8GiB | 42.3% done | ETA(epoch): 889s + [Epoch 211][30/66] loss=0.829990 avg=0.836588 VRAM=39.0GiB | 42.3% done | ETA(epoch): 695s + [Epoch 211][40/66] loss=0.859942 avg=0.845272 VRAM=38.8GiB | 42.3% done | ETA(epoch): 501s + [Epoch 211][50/66] loss=0.873429 avg=0.846288 VRAM=39.0GiB | 42.4% done | ETA(epoch): 308s + [Epoch 211][60/66] loss=0.789103 avg=0.842800 VRAM=38.8GiB | 42.4% done | ETA(epoch): 116s + Train loss: 0.844405 (1271.7s) ETA: 6654min + Val loss: 0.827774 [t_0.0-0.2=1.0541 t_0.2-0.4=1.0034 t_0.4-0.6=0.8019 t_0.6-0.8=0.7012 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 212/499 (42% done) --- + [Epoch 212][10/66] loss=0.755565 avg=0.853821 VRAM=39.0GiB | 42.4% done | ETA(epoch): 1075s + [Epoch 212][20/66] loss=0.780752 avg=0.869102 VRAM=38.8GiB | 42.5% done | ETA(epoch): 884s + [Epoch 212][30/66] loss=0.927556 avg=0.862558 VRAM=39.0GiB | 42.5% done | ETA(epoch): 692s + [Epoch 212][40/66] loss=0.828916 avg=0.858072 VRAM=38.8GiB | 42.5% done | ETA(epoch): 500s + [Epoch 212][50/66] loss=0.726918 avg=0.851973 VRAM=39.0GiB | 42.6% done | ETA(epoch): 308s + [Epoch 212][60/66] loss=0.905507 avg=0.850808 VRAM=38.8GiB | 42.6% done | ETA(epoch): 115s + Train loss: 0.845805 (1268.4s) ETA: 6631min + Val loss: 0.861182 [t_0.0-0.2=1.0829 t_0.2-0.4=0.9798 t_0.4-0.6=0.8410 t_0.6-0.8=0.7068 t_0.8-1.0=0.6285] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 213/499 (43% done) --- + [Epoch 213][10/66] loss=0.832019 avg=0.853027 VRAM=39.0GiB | 42.6% done | ETA(epoch): 1090s + [Epoch 213][20/66] loss=0.804652 avg=0.854645 VRAM=38.8GiB | 42.7% done | ETA(epoch): 890s + [Epoch 213][30/66] loss=0.847724 avg=0.858985 VRAM=39.0GiB | 42.7% done | ETA(epoch): 695s + [Epoch 213][40/66] loss=0.843162 avg=0.858087 VRAM=38.8GiB | 42.7% done | ETA(epoch): 501s + [Epoch 213][50/66] loss=0.922994 avg=0.862382 VRAM=39.0GiB | 42.8% done | ETA(epoch): 308s + [Epoch 213][60/66] loss=0.982048 avg=0.869410 VRAM=38.8GiB | 42.8% done | ETA(epoch): 116s + Train loss: 0.866669 (1270.5s) ETA: 6607min + Val loss: 0.866638 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9895 t_0.4-0.6=0.8543 t_0.6-0.8=0.7072 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 214/499 (43% done) --- + [Epoch 214][10/66] loss=0.716699 avg=0.825609 VRAM=39.0GiB | 42.8% done | ETA(epoch): 1075s + [Epoch 214][20/66] loss=0.831642 avg=0.837011 VRAM=38.8GiB | 42.9% done | ETA(epoch): 883s + [Epoch 214][30/66] loss=0.803909 avg=0.837684 VRAM=39.0GiB | 42.9% done | ETA(epoch): 692s + [Epoch 214][40/66] loss=0.935980 avg=0.835139 VRAM=38.8GiB | 42.9% done | ETA(epoch): 500s + [Epoch 214][50/66] loss=0.822461 avg=0.843532 VRAM=39.0GiB | 43.0% done | ETA(epoch): 307s + [Epoch 214][60/66] loss=0.900537 avg=0.845117 VRAM=38.8GiB | 43.0% done | ETA(epoch): 115s + Train loss: 0.845099 (1268.0s) ETA: 6583min + Val loss: 0.862103 [t_0.0-0.2=1.0642 t_0.2-0.4=0.9940 t_0.4-0.6=0.8527 t_0.6-0.8=0.7388 t_0.8-1.0=0.6304] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 215/499 (43% done) --- + [Epoch 215][10/66] loss=0.839886 avg=0.823318 VRAM=39.0GiB | 43.0% done | ETA(epoch): 1076s + [Epoch 215][20/66] loss=0.966900 avg=0.822344 VRAM=38.8GiB | 43.1% done | ETA(epoch): 884s + [Epoch 215][30/66] loss=0.890748 avg=0.828292 VRAM=39.0GiB | 43.1% done | ETA(epoch): 692s + [Epoch 215][40/66] loss=0.816539 avg=0.836781 VRAM=38.8GiB | 43.1% done | ETA(epoch): 500s + [Epoch 215][50/66] loss=1.019281 avg=0.840050 VRAM=39.0GiB | 43.2% done | ETA(epoch): 308s + [Epoch 215][60/66] loss=0.858796 avg=0.844087 VRAM=38.8GiB | 43.2% done | ETA(epoch): 115s + Train loss: 0.840659 (1270.3s) ETA: 6560min + Val loss: 0.851393 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0217 t_0.4-0.6=0.8050 t_0.6-0.8=0.7001 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 216/499 (43% done) --- + [Epoch 216][10/66] loss=0.752352 avg=0.835511 VRAM=39.0GiB | 43.2% done | ETA(epoch): 1076s + [Epoch 216][20/66] loss=0.702814 avg=0.836170 VRAM=38.8GiB | 43.3% done | ETA(epoch): 884s + [Epoch 216][30/66] loss=0.958836 avg=0.845084 VRAM=39.0GiB | 43.3% done | ETA(epoch): 691s + [Epoch 216][40/66] loss=0.885586 avg=0.846056 VRAM=38.8GiB | 43.3% done | ETA(epoch): 504s + [Epoch 216][50/66] loss=0.917290 avg=0.837443 VRAM=39.0GiB | 43.4% done | ETA(epoch): 310s + [Epoch 216][60/66] loss=0.939816 avg=0.835582 VRAM=38.8GiB | 43.4% done | ETA(epoch): 116s + Train loss: 0.838592 (1274.8s) ETA: 6536min + Val loss: 0.863957 [t_0.0-0.2=1.0637 t_0.2-0.4=1.0266 t_0.4-0.6=0.8330 t_0.6-0.8=0.7260 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 217/499 (43% done) --- + [Epoch 217][10/66] loss=0.966928 avg=0.876418 VRAM=39.0GiB | 43.4% done | ETA(epoch): 1074s + [Epoch 217][20/66] loss=0.991782 avg=0.864737 VRAM=38.8GiB | 43.5% done | ETA(epoch): 883s + [Epoch 217][30/66] loss=0.857168 avg=0.849551 VRAM=39.0GiB | 43.5% done | ETA(epoch): 691s + [Epoch 217][40/66] loss=0.795394 avg=0.844853 VRAM=38.8GiB | 43.5% done | ETA(epoch): 499s + [Epoch 217][50/66] loss=0.930431 avg=0.843300 VRAM=39.0GiB | 43.6% done | ETA(epoch): 307s + [Epoch 217][60/66] loss=0.939510 avg=0.849511 VRAM=38.8GiB | 43.6% done | ETA(epoch): 115s + Train loss: 0.848648 (1267.7s) ETA: 6513min + Val loss: 0.850127 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9767 t_0.4-0.6=0.8120 t_0.6-0.8=0.7366 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 218/499 (44% done) --- + [Epoch 218][10/66] loss=0.718677 avg=0.871786 VRAM=39.0GiB | 43.6% done | ETA(epoch): 1075s + [Epoch 218][20/66] loss=0.870697 avg=0.870225 VRAM=38.8GiB | 43.7% done | ETA(epoch): 883s + [Epoch 218][30/66] loss=0.849146 avg=0.863849 VRAM=39.0GiB | 43.7% done | ETA(epoch): 694s + [Epoch 218][40/66] loss=0.974923 avg=0.868625 VRAM=38.8GiB | 43.7% done | ETA(epoch): 501s + [Epoch 218][50/66] loss=0.992555 avg=0.869475 VRAM=39.0GiB | 43.8% done | ETA(epoch): 308s + [Epoch 218][60/66] loss=0.834050 avg=0.867577 VRAM=38.8GiB | 43.8% done | ETA(epoch): 115s + Train loss: 0.866140 (1269.2s) ETA: 6489min + Val loss: 0.865024 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9842 t_0.4-0.6=0.8181 t_0.6-0.8=0.7068 t_0.8-1.0=0.6288] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 219/499 (44% done) --- + [Epoch 219][10/66] loss=0.892850 avg=0.883983 VRAM=39.0GiB | 43.8% done | ETA(epoch): 1073s + [Epoch 219][20/66] loss=0.728304 avg=0.852068 VRAM=38.8GiB | 43.9% done | ETA(epoch): 891s + [Epoch 219][30/66] loss=0.778612 avg=0.846417 VRAM=39.0GiB | 43.9% done | ETA(epoch): 696s + [Epoch 219][40/66] loss=0.855670 avg=0.852162 VRAM=38.8GiB | 43.9% done | ETA(epoch): 502s + [Epoch 219][50/66] loss=0.817492 avg=0.842162 VRAM=39.0GiB | 44.0% done | ETA(epoch): 308s + [Epoch 219][60/66] loss=0.805690 avg=0.841962 VRAM=38.8GiB | 44.0% done | ETA(epoch): 116s + Train loss: 0.845381 (1270.7s) ETA: 6466min + Val loss: 0.891979 [t_0.0-0.2=1.0528 t_0.2-0.4=1.0247 t_0.4-0.6=0.8548 t_0.6-0.8=0.7138 t_0.8-1.0=0.6193] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 220/499 (44% done) --- + [Epoch 220][10/66] loss=0.830745 avg=0.888843 VRAM=39.0GiB | 44.0% done | ETA(epoch): 1077s + [Epoch 220][20/66] loss=0.828985 avg=0.885751 VRAM=38.8GiB | 44.1% done | ETA(epoch): 884s + [Epoch 220][30/66] loss=0.917688 avg=0.875548 VRAM=39.0GiB | 44.1% done | ETA(epoch): 691s + [Epoch 220][40/66] loss=0.703971 avg=0.865249 VRAM=38.8GiB | 44.1% done | ETA(epoch): 499s + [Epoch 220][50/66] loss=0.935791 avg=0.860162 VRAM=39.0GiB | 44.2% done | ETA(epoch): 307s + [Epoch 220][60/66] loss=0.712527 avg=0.858323 VRAM=38.8GiB | 44.2% done | ETA(epoch): 115s + Train loss: 0.856790 (1266.7s) ETA: 6442min + Val loss: 0.819137 [t_0.0-0.2=1.0858 t_0.2-0.4=0.9736 t_0.4-0.6=0.8337 t_0.6-0.8=0.7088 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 221/499 (44% done) --- + [Epoch 221][10/66] loss=0.727500 avg=0.812764 VRAM=39.0GiB | 44.2% done | ETA(epoch): 1074s + [Epoch 221][20/66] loss=0.905444 avg=0.831634 VRAM=38.8GiB | 44.3% done | ETA(epoch): 883s + [Epoch 221][30/66] loss=0.863493 avg=0.841747 VRAM=39.0GiB | 44.3% done | ETA(epoch): 691s + [Epoch 221][40/66] loss=0.751137 avg=0.846653 VRAM=38.8GiB | 44.3% done | ETA(epoch): 499s + [Epoch 221][50/66] loss=0.886278 avg=0.856675 VRAM=39.0GiB | 44.4% done | ETA(epoch): 307s + [Epoch 221][60/66] loss=0.783621 avg=0.850554 VRAM=38.8GiB | 44.4% done | ETA(epoch): 115s + Train loss: 0.846428 (1266.4s) ETA: 6418min + Val loss: 0.842119 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9850 t_0.4-0.6=0.8314 t_0.6-0.8=0.7049 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 222/499 (44% done) --- + [Epoch 222][10/66] loss=0.947687 avg=0.871549 VRAM=39.0GiB | 44.4% done | ETA(epoch): 1074s + [Epoch 222][20/66] loss=1.026120 avg=0.852107 VRAM=38.8GiB | 44.5% done | ETA(epoch): 882s + [Epoch 222][30/66] loss=0.728742 avg=0.848847 VRAM=39.0GiB | 44.5% done | ETA(epoch): 691s + [Epoch 222][40/66] loss=0.975253 avg=0.855924 VRAM=38.8GiB | 44.5% done | ETA(epoch): 499s + [Epoch 222][50/66] loss=0.855720 avg=0.856162 VRAM=39.0GiB | 44.6% done | ETA(epoch): 307s + [Epoch 222][60/66] loss=0.672259 avg=0.860108 VRAM=38.8GiB | 44.6% done | ETA(epoch): 115s + Train loss: 0.857070 (1266.2s) ETA: 6395min + Val loss: 0.837082 [t_0.0-0.2=1.0703 t_0.2-0.4=0.9475 t_0.4-0.6=0.8547 t_0.6-0.8=0.6989 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 223/499 (45% done) --- + [Epoch 223][10/66] loss=0.906751 avg=0.857135 VRAM=39.0GiB | 44.6% done | ETA(epoch): 1072s + [Epoch 223][20/66] loss=0.754543 avg=0.852901 VRAM=38.8GiB | 44.7% done | ETA(epoch): 880s + [Epoch 223][30/66] loss=0.791490 avg=0.841761 VRAM=39.0GiB | 44.7% done | ETA(epoch): 689s + [Epoch 223][40/66] loss=0.764487 avg=0.840679 VRAM=38.8GiB | 44.7% done | ETA(epoch): 498s + [Epoch 223][50/66] loss=0.851040 avg=0.840327 VRAM=39.0GiB | 44.8% done | ETA(epoch): 306s + [Epoch 223][60/66] loss=0.808566 avg=0.841058 VRAM=38.8GiB | 44.8% done | ETA(epoch): 115s + Train loss: 0.837291 (1263.6s) ETA: 6371min + Val loss: 0.866507 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9793 t_0.4-0.6=0.8547 t_0.6-0.8=0.7111 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 224/499 (45% done) --- + [Epoch 224][10/66] loss=0.724592 avg=0.812747 VRAM=39.0GiB | 44.8% done | ETA(epoch): 1076s + [Epoch 224][20/66] loss=0.858747 avg=0.831078 VRAM=38.8GiB | 44.9% done | ETA(epoch): 883s + [Epoch 224][30/66] loss=0.822930 avg=0.848866 VRAM=39.0GiB | 44.9% done | ETA(epoch): 691s + [Epoch 224][40/66] loss=0.801446 avg=0.846057 VRAM=38.8GiB | 44.9% done | ETA(epoch): 499s + [Epoch 224][50/66] loss=0.973431 avg=0.852863 VRAM=39.0GiB | 45.0% done | ETA(epoch): 307s + [Epoch 224][60/66] loss=0.769720 avg=0.847538 VRAM=38.8GiB | 45.0% done | ETA(epoch): 115s + Train loss: 0.843794 (1265.5s) ETA: 6348min + Val loss: 0.840426 [t_0.0-0.2=1.0853 t_0.2-0.4=1.0032 t_0.4-0.6=0.8395 t_0.6-0.8=0.6909 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 225/499 (45% done) --- + [Epoch 225][10/66] loss=0.882315 avg=0.836923 VRAM=39.0GiB | 45.0% done | ETA(epoch): 1073s + [Epoch 225][20/66] loss=0.681139 avg=0.845834 VRAM=38.8GiB | 45.1% done | ETA(epoch): 881s + [Epoch 225][30/66] loss=0.821406 avg=0.845114 VRAM=39.0GiB | 45.1% done | ETA(epoch): 694s + [Epoch 225][40/66] loss=0.869905 avg=0.846312 VRAM=38.8GiB | 45.1% done | ETA(epoch): 501s + [Epoch 225][50/66] loss=0.803230 avg=0.847889 VRAM=39.0GiB | 45.2% done | ETA(epoch): 308s + [Epoch 225][60/66] loss=0.711802 avg=0.849257 VRAM=38.8GiB | 45.2% done | ETA(epoch): 115s + Train loss: 0.845615 (1268.4s) ETA: 6324min + Val loss: 0.820079 [t_0.0-0.2=1.0773 t_0.2-0.4=0.9980 t_0.4-0.6=0.8210 t_0.6-0.8=0.7019 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 226/499 (45% done) --- + [Epoch 226][10/66] loss=0.835588 avg=0.805741 VRAM=39.0GiB | 45.2% done | ETA(epoch): 1073s + [Epoch 226][20/66] loss=0.886537 avg=0.830797 VRAM=38.8GiB | 45.3% done | ETA(epoch): 881s + [Epoch 226][30/66] loss=0.972434 avg=0.833275 VRAM=39.0GiB | 45.3% done | ETA(epoch): 690s + [Epoch 226][40/66] loss=0.847510 avg=0.834595 VRAM=38.8GiB | 45.3% done | ETA(epoch): 498s + [Epoch 226][50/66] loss=0.890351 avg=0.835533 VRAM=39.0GiB | 45.4% done | ETA(epoch): 307s + [Epoch 226][60/66] loss=0.783370 avg=0.836630 VRAM=38.8GiB | 45.4% done | ETA(epoch): 115s + Train loss: 0.836078 (1265.1s) ETA: 6300min + Val loss: 0.854336 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9915 t_0.4-0.6=0.8549 t_0.6-0.8=0.7294 t_0.8-1.0=0.6189] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 227/499 (45% done) --- + [Epoch 227][10/66] loss=0.855779 avg=0.841296 VRAM=39.0GiB | 45.4% done | ETA(epoch): 1073s + [Epoch 227][20/66] loss=0.964975 avg=0.832435 VRAM=38.8GiB | 45.5% done | ETA(epoch): 882s + [Epoch 227][30/66] loss=0.823901 avg=0.845958 VRAM=39.0GiB | 45.5% done | ETA(epoch): 691s + [Epoch 227][40/66] loss=0.688285 avg=0.840681 VRAM=38.8GiB | 45.5% done | ETA(epoch): 499s + [Epoch 227][50/66] loss=0.916162 avg=0.843788 VRAM=39.0GiB | 45.6% done | ETA(epoch): 307s + [Epoch 227][60/66] loss=0.891784 avg=0.839939 VRAM=38.8GiB | 45.6% done | ETA(epoch): 115s + Train loss: 0.844125 (1266.6s) ETA: 6277min + Val loss: 0.827402 [t_0.0-0.2=1.0590 t_0.2-0.4=0.9955 t_0.4-0.6=0.8777 t_0.6-0.8=0.6984 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 228/499 (46% done) --- + [Epoch 228][10/66] loss=0.826280 avg=0.875351 VRAM=39.0GiB | 45.6% done | ETA(epoch): 1074s + [Epoch 228][20/66] loss=0.923696 avg=0.864230 VRAM=38.8GiB | 45.7% done | ETA(epoch): 882s + [Epoch 228][30/66] loss=0.951424 avg=0.865626 VRAM=39.0GiB | 45.7% done | ETA(epoch): 690s + [Epoch 228][40/66] loss=0.782842 avg=0.868047 VRAM=38.8GiB | 45.7% done | ETA(epoch): 499s + [Epoch 228][50/66] loss=0.844948 avg=0.861688 VRAM=39.0GiB | 45.8% done | ETA(epoch): 307s + [Epoch 228][60/66] loss=0.815164 avg=0.857050 VRAM=38.8GiB | 45.8% done | ETA(epoch): 115s + Train loss: 0.851293 (1266.1s) ETA: 6253min + Val loss: 0.862384 [t_0.0-0.2=1.0645 t_0.2-0.4=1.0058 t_0.4-0.6=0.8487 t_0.6-0.8=0.6797 t_0.8-1.0=0.6142] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 229/499 (46% done) --- + [Epoch 229][10/66] loss=0.908736 avg=0.834824 VRAM=39.0GiB | 45.8% done | ETA(epoch): 1074s + [Epoch 229][20/66] loss=0.864295 avg=0.835338 VRAM=38.8GiB | 45.9% done | ETA(epoch): 882s + [Epoch 229][30/66] loss=0.892894 avg=0.839568 VRAM=39.0GiB | 45.9% done | ETA(epoch): 690s + [Epoch 229][40/66] loss=0.825165 avg=0.844057 VRAM=38.8GiB | 45.9% done | ETA(epoch): 499s + [Epoch 229][50/66] loss=0.873480 avg=0.852399 VRAM=39.0GiB | 46.0% done | ETA(epoch): 307s + [Epoch 229][60/66] loss=0.847618 avg=0.848442 VRAM=38.8GiB | 46.0% done | ETA(epoch): 115s + Train loss: 0.848117 (1266.5s) ETA: 6230min + Val loss: 0.864271 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9933 t_0.4-0.6=0.8496 t_0.6-0.8=0.6893 t_0.8-1.0=0.6385] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 230/499 (46% done) --- + [Epoch 230][10/66] loss=0.897903 avg=0.880597 VRAM=39.0GiB | 46.0% done | ETA(epoch): 1076s + [Epoch 230][20/66] loss=0.821108 avg=0.859297 VRAM=38.8GiB | 46.1% done | ETA(epoch): 883s + [Epoch 230][30/66] loss=0.941227 avg=0.851746 VRAM=39.0GiB | 46.1% done | ETA(epoch): 691s + [Epoch 230][40/66] loss=0.884528 avg=0.862404 VRAM=38.8GiB | 46.1% done | ETA(epoch): 499s + [Epoch 230][50/66] loss=0.872263 avg=0.863109 VRAM=39.0GiB | 46.2% done | ETA(epoch): 307s + [Epoch 230][60/66] loss=0.908492 avg=0.867374 VRAM=38.8GiB | 46.2% done | ETA(epoch): 115s + Train loss: 0.864156 (1266.4s) ETA: 6206min + Val loss: 0.832433 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9973 t_0.4-0.6=0.8161 t_0.6-0.8=0.6936 t_0.8-1.0=0.6108] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 231/499 (46% done) --- + [Epoch 231][10/66] loss=0.862840 avg=0.802095 VRAM=39.0GiB | 46.2% done | ETA(epoch): 1074s + [Epoch 231][20/66] loss=0.867809 avg=0.837781 VRAM=38.8GiB | 46.3% done | ETA(epoch): 883s + [Epoch 231][30/66] loss=0.881149 avg=0.842822 VRAM=39.0GiB | 46.3% done | ETA(epoch): 691s + [Epoch 231][40/66] loss=0.804327 avg=0.832795 VRAM=38.8GiB | 46.3% done | ETA(epoch): 499s + [Epoch 231][50/66] loss=0.879875 avg=0.837121 VRAM=39.0GiB | 46.4% done | ETA(epoch): 308s + [Epoch 231][60/66] loss=0.961782 avg=0.841421 VRAM=38.8GiB | 46.4% done | ETA(epoch): 115s + Train loss: 0.837652 (1268.1s) ETA: 6183min + Val loss: 0.848290 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9594 t_0.4-0.6=0.8415 t_0.6-0.8=0.7191 t_0.8-1.0=0.6158] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 232/499 (46% done) --- + [Epoch 232][10/66] loss=0.808109 avg=0.858271 VRAM=39.0GiB | 46.4% done | ETA(epoch): 1074s + [Epoch 232][20/66] loss=0.962815 avg=0.865166 VRAM=38.8GiB | 46.5% done | ETA(epoch): 883s + [Epoch 232][30/66] loss=0.927532 avg=0.858917 VRAM=39.0GiB | 46.5% done | ETA(epoch): 691s + [Epoch 232][40/66] loss=0.981595 avg=0.867625 VRAM=38.8GiB | 46.5% done | ETA(epoch): 499s + [Epoch 232][50/66] loss=0.728498 avg=0.862699 VRAM=39.0GiB | 46.6% done | ETA(epoch): 307s + [Epoch 232][60/66] loss=0.954551 avg=0.863204 VRAM=38.8GiB | 46.6% done | ETA(epoch): 115s + Train loss: 0.860860 (1266.6s) ETA: 6159min + Val loss: 0.817653 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9738 t_0.4-0.6=0.8075 t_0.6-0.8=0.6873 t_0.8-1.0=0.6318] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 233/499 (47% done) --- + [Epoch 233][10/66] loss=0.798826 avg=0.844617 VRAM=39.0GiB | 46.6% done | ETA(epoch): 1072s + [Epoch 233][20/66] loss=0.901533 avg=0.810028 VRAM=38.8GiB | 46.7% done | ETA(epoch): 881s + [Epoch 233][30/66] loss=0.852935 avg=0.828363 VRAM=39.0GiB | 46.7% done | ETA(epoch): 690s + [Epoch 233][40/66] loss=0.972663 avg=0.833822 VRAM=38.8GiB | 46.7% done | ETA(epoch): 498s + [Epoch 233][50/66] loss=0.802058 avg=0.839479 VRAM=39.0GiB | 46.8% done | ETA(epoch): 307s + [Epoch 233][60/66] loss=0.748202 avg=0.844252 VRAM=38.8GiB | 46.8% done | ETA(epoch): 115s + Train loss: 0.846940 (1265.3s) ETA: 6136min + Val loss: 0.841515 [t_0.0-0.2=1.0580 t_0.2-0.4=0.9974 t_0.4-0.6=0.8376 t_0.6-0.8=0.7234 t_0.8-1.0=0.6416] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 234/499 (47% done) --- + [Epoch 234][10/66] loss=0.758914 avg=0.828955 VRAM=39.0GiB | 46.8% done | ETA(epoch): 1075s + [Epoch 234][20/66] loss=0.813117 avg=0.861926 VRAM=38.8GiB | 46.9% done | ETA(epoch): 882s + [Epoch 234][30/66] loss=0.713456 avg=0.848294 VRAM=39.0GiB | 46.9% done | ETA(epoch): 691s + [Epoch 234][40/66] loss=0.901606 avg=0.855714 VRAM=38.8GiB | 46.9% done | ETA(epoch): 499s + [Epoch 234][50/66] loss=0.955660 avg=0.854499 VRAM=39.0GiB | 47.0% done | ETA(epoch): 307s + [Epoch 234][60/66] loss=0.866541 avg=0.858002 VRAM=38.8GiB | 47.0% done | ETA(epoch): 115s + Train loss: 0.858819 (1266.1s) ETA: 6112min + Val loss: 0.809256 [t_0.0-0.2=1.0701 t_0.2-0.4=1.0219 t_0.4-0.6=0.8580 t_0.6-0.8=0.6972 t_0.8-1.0=0.6254] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 235/499 (47% done) --- + [Epoch 235][10/66] loss=0.717858 avg=0.748159 VRAM=39.0GiB | 47.0% done | ETA(epoch): 1085s + [Epoch 235][20/66] loss=0.829765 avg=0.797529 VRAM=38.8GiB | 47.1% done | ETA(epoch): 886s + [Epoch 235][30/66] loss=0.825034 avg=0.799400 VRAM=39.0GiB | 47.1% done | ETA(epoch): 692s + [Epoch 235][40/66] loss=0.975812 avg=0.818746 VRAM=38.8GiB | 47.1% done | ETA(epoch): 500s + [Epoch 235][50/66] loss=0.877342 avg=0.828360 VRAM=39.0GiB | 47.2% done | ETA(epoch): 307s + [Epoch 235][60/66] loss=0.900777 avg=0.838865 VRAM=38.8GiB | 47.2% done | ETA(epoch): 115s + Train loss: 0.836985 (1267.4s) ETA: 6089min + Val loss: 0.830581 [t_0.0-0.2=1.0685 t_0.2-0.4=0.9846 t_0.4-0.6=0.8349 t_0.6-0.8=0.7072 t_0.8-1.0=0.6142] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 236/499 (47% done) --- + [Epoch 236][10/66] loss=0.877710 avg=0.828578 VRAM=39.0GiB | 47.2% done | ETA(epoch): 1085s + [Epoch 236][20/66] loss=0.852024 avg=0.832057 VRAM=38.8GiB | 47.3% done | ETA(epoch): 886s + [Epoch 236][30/66] loss=0.834148 avg=0.836629 VRAM=39.0GiB | 47.3% done | ETA(epoch): 693s + [Epoch 236][40/66] loss=0.793286 avg=0.840881 VRAM=38.8GiB | 47.3% done | ETA(epoch): 500s + [Epoch 236][50/66] loss=0.691016 avg=0.836330 VRAM=39.0GiB | 47.4% done | ETA(epoch): 307s + [Epoch 236][60/66] loss=0.836656 avg=0.835857 VRAM=38.8GiB | 47.4% done | ETA(epoch): 115s + Train loss: 0.840364 (1267.9s) ETA: 6065min + Val loss: 0.816834 [t_0.0-0.2=1.0804 t_0.2-0.4=0.9654 t_0.4-0.6=0.8403 t_0.6-0.8=0.7228 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 237/499 (47% done) --- + [Epoch 237][10/66] loss=0.850432 avg=0.863853 VRAM=39.0GiB | 47.4% done | ETA(epoch): 1074s + [Epoch 237][20/66] loss=0.882424 avg=0.856066 VRAM=38.8GiB | 47.5% done | ETA(epoch): 883s + [Epoch 237][30/66] loss=1.021373 avg=0.864609 VRAM=39.0GiB | 47.5% done | ETA(epoch): 691s + [Epoch 237][40/66] loss=0.984356 avg=0.867618 VRAM=38.8GiB | 47.5% done | ETA(epoch): 499s + [Epoch 237][50/66] loss=0.736527 avg=0.865408 VRAM=39.0GiB | 47.6% done | ETA(epoch): 307s + [Epoch 237][60/66] loss=0.708171 avg=0.861100 VRAM=38.8GiB | 47.6% done | ETA(epoch): 115s + Train loss: 0.855982 (1266.5s) ETA: 6042min + Val loss: 0.879908 [t_0.0-0.2=1.0736 t_0.2-0.4=1.0012 t_0.4-0.6=0.8481 t_0.6-0.8=0.6925 t_0.8-1.0=0.6215] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 238/499 (48% done) --- + [Epoch 238][10/66] loss=0.958000 avg=0.864346 VRAM=39.0GiB | 47.6% done | ETA(epoch): 1073s + [Epoch 238][20/66] loss=0.915150 avg=0.859192 VRAM=38.8GiB | 47.7% done | ETA(epoch): 882s + [Epoch 238][30/66] loss=0.869126 avg=0.849527 VRAM=39.0GiB | 47.7% done | ETA(epoch): 691s + [Epoch 238][40/66] loss=0.873866 avg=0.835360 VRAM=38.8GiB | 47.7% done | ETA(epoch): 499s + [Epoch 238][50/66] loss=0.804881 avg=0.841234 VRAM=39.0GiB | 47.8% done | ETA(epoch): 308s + [Epoch 238][60/66] loss=0.880405 avg=0.843396 VRAM=38.8GiB | 47.8% done | ETA(epoch): 115s + Train loss: 0.839742 (1268.2s) ETA: 6018min + Val loss: 0.848546 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9966 t_0.4-0.6=0.8262 t_0.6-0.8=0.7061 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 239/499 (48% done) --- + [Epoch 239][10/66] loss=0.906406 avg=0.862344 VRAM=39.0GiB | 47.8% done | ETA(epoch): 1074s + [Epoch 239][20/66] loss=0.777465 avg=0.873967 VRAM=38.8GiB | 47.9% done | ETA(epoch): 882s + [Epoch 239][30/66] loss=0.815287 avg=0.868556 VRAM=39.0GiB | 47.9% done | ETA(epoch): 691s + [Epoch 239][40/66] loss=0.890219 avg=0.872409 VRAM=38.8GiB | 47.9% done | ETA(epoch): 501s + [Epoch 239][50/66] loss=0.858380 avg=0.869222 VRAM=39.0GiB | 48.0% done | ETA(epoch): 308s + [Epoch 239][60/66] loss=0.833630 avg=0.867372 VRAM=38.8GiB | 48.0% done | ETA(epoch): 115s + Train loss: 0.869391 (1269.8s) ETA: 5995min + Val loss: 0.839895 [t_0.0-0.2=1.0808 t_0.2-0.4=1.0011 t_0.4-0.6=0.8311 t_0.6-0.8=0.7019 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 240/499 (48% done) --- + [Epoch 240][10/66] loss=0.916483 avg=0.857512 VRAM=39.0GiB | 48.0% done | ETA(epoch): 1074s + [Epoch 240][20/66] loss=0.832368 avg=0.859069 VRAM=38.8GiB | 48.1% done | ETA(epoch): 882s + [Epoch 240][30/66] loss=0.779351 avg=0.848427 VRAM=39.0GiB | 48.1% done | ETA(epoch): 690s + [Epoch 240][40/66] loss=0.805296 avg=0.858208 VRAM=38.8GiB | 48.1% done | ETA(epoch): 500s + [Epoch 240][50/66] loss=0.903094 avg=0.865442 VRAM=39.0GiB | 48.2% done | ETA(epoch): 308s + [Epoch 240][60/66] loss=0.886189 avg=0.859459 VRAM=38.8GiB | 48.2% done | ETA(epoch): 115s + Train loss: 0.859474 (1268.6s) ETA: 5972min + Val loss: 0.855245 [t_0.0-0.2=1.0821 t_0.2-0.4=0.9930 t_0.4-0.6=0.8235 t_0.6-0.8=0.7151 t_0.8-1.0=0.6132] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 241/499 (48% done) --- + [Epoch 241][10/66] loss=0.803909 avg=0.832737 VRAM=39.0GiB | 48.2% done | ETA(epoch): 1074s + [Epoch 241][20/66] loss=0.808087 avg=0.832422 VRAM=38.8GiB | 48.3% done | ETA(epoch): 882s + [Epoch 241][30/66] loss=0.859062 avg=0.850566 VRAM=39.0GiB | 48.3% done | ETA(epoch): 693s + [Epoch 241][40/66] loss=0.884797 avg=0.847426 VRAM=38.8GiB | 48.3% done | ETA(epoch): 500s + [Epoch 241][50/66] loss=0.973229 avg=0.849745 VRAM=39.0GiB | 48.4% done | ETA(epoch): 308s + [Epoch 241][60/66] loss=0.867867 avg=0.848053 VRAM=38.8GiB | 48.4% done | ETA(epoch): 115s + Train loss: 0.848945 (1268.3s) ETA: 5948min + Val loss: 0.850065 [t_0.0-0.2=1.0729 t_0.2-0.4=0.9804 t_0.4-0.6=0.8543 t_0.6-0.8=0.6937 t_0.8-1.0=0.6502] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 242/499 (48% done) --- + [Epoch 242][10/66] loss=0.696807 avg=0.813941 VRAM=39.0GiB | 48.4% done | ETA(epoch): 1074s + [Epoch 242][20/66] loss=0.775589 avg=0.831198 VRAM=38.8GiB | 48.5% done | ETA(epoch): 882s + [Epoch 242][30/66] loss=0.864204 avg=0.826493 VRAM=39.0GiB | 48.5% done | ETA(epoch): 690s + [Epoch 242][40/66] loss=0.805857 avg=0.831370 VRAM=38.8GiB | 48.5% done | ETA(epoch): 499s + [Epoch 242][50/66] loss=0.931756 avg=0.838646 VRAM=39.0GiB | 48.6% done | ETA(epoch): 307s + [Epoch 242][60/66] loss=0.756818 avg=0.837810 VRAM=38.8GiB | 48.6% done | ETA(epoch): 115s + Train loss: 0.840204 (1265.8s) ETA: 5925min + Val loss: 0.835034 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9478 t_0.4-0.6=0.8953 t_0.6-0.8=0.7175 t_0.8-1.0=0.6252] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 243/499 (49% done) --- + [Epoch 243][10/66] loss=0.844570 avg=0.818464 VRAM=39.0GiB | 48.6% done | ETA(epoch): 1088s + [Epoch 243][20/66] loss=0.869376 avg=0.808414 VRAM=38.8GiB | 48.7% done | ETA(epoch): 887s + [Epoch 243][30/66] loss=0.765161 avg=0.810482 VRAM=39.0GiB | 48.7% done | ETA(epoch): 693s + [Epoch 243][40/66] loss=0.806267 avg=0.815352 VRAM=38.8GiB | 48.7% done | ETA(epoch): 500s + [Epoch 243][50/66] loss=0.915829 avg=0.827766 VRAM=39.0GiB | 48.8% done | ETA(epoch): 308s + [Epoch 243][60/66] loss=0.867781 avg=0.832323 VRAM=38.8GiB | 48.8% done | ETA(epoch): 115s + Train loss: 0.837553 (1268.6s) ETA: 5901min + Val loss: 0.844638 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9661 t_0.4-0.6=0.8526 t_0.6-0.8=0.7065 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 244/499 (49% done) --- + [Epoch 244][10/66] loss=1.017699 avg=0.843554 VRAM=39.0GiB | 48.8% done | ETA(epoch): 1073s + [Epoch 244][20/66] loss=0.966598 avg=0.833864 VRAM=38.8GiB | 48.9% done | ETA(epoch): 882s + [Epoch 244][30/66] loss=1.054569 avg=0.839786 VRAM=39.0GiB | 48.9% done | ETA(epoch): 690s + [Epoch 244][40/66] loss=0.962733 avg=0.845077 VRAM=38.8GiB | 48.9% done | ETA(epoch): 499s + [Epoch 244][50/66] loss=0.937184 avg=0.842076 VRAM=39.0GiB | 49.0% done | ETA(epoch): 307s + [Epoch 244][60/66] loss=0.805984 avg=0.841337 VRAM=38.8GiB | 49.0% done | ETA(epoch): 115s + Train loss: 0.845167 (1265.8s) ETA: 5878min + Val loss: 0.863942 [t_0.0-0.2=1.0530 t_0.2-0.4=0.9861 t_0.4-0.6=0.8450 t_0.6-0.8=0.7163 t_0.8-1.0=0.6336] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 245/499 (49% done) --- + [Epoch 245][10/66] loss=0.839737 avg=0.855149 VRAM=39.0GiB | 49.0% done | ETA(epoch): 1073s + [Epoch 245][20/66] loss=0.897669 avg=0.851400 VRAM=38.8GiB | 49.1% done | ETA(epoch): 883s + [Epoch 245][30/66] loss=0.961602 avg=0.865118 VRAM=39.0GiB | 49.1% done | ETA(epoch): 691s + [Epoch 245][40/66] loss=0.976506 avg=0.857215 VRAM=38.8GiB | 49.1% done | ETA(epoch): 499s + [Epoch 245][50/66] loss=0.967618 avg=0.855579 VRAM=39.0GiB | 49.2% done | ETA(epoch): 307s + [Epoch 245][60/66] loss=0.783367 avg=0.850093 VRAM=38.8GiB | 49.2% done | ETA(epoch): 117s + Train loss: 0.852061 (1284.3s) ETA: 5855min + Val loss: 0.801178 [t_0.0-0.2=1.0606 t_0.2-0.4=1.0217 t_0.4-0.6=0.8350 t_0.6-0.8=0.6834 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0245 (BEST) + Deleted old checkpoint: checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 246/499 (49% done) --- + [Epoch 246][10/66] loss=0.806694 avg=0.845011 VRAM=39.0GiB | 49.2% done | ETA(epoch): 1074s + [Epoch 246][20/66] loss=0.876609 avg=0.833815 VRAM=38.8GiB | 49.3% done | ETA(epoch): 883s + [Epoch 246][30/66] loss=0.857570 avg=0.833330 VRAM=39.0GiB | 49.3% done | ETA(epoch): 691s + [Epoch 246][40/66] loss=0.819935 avg=0.845798 VRAM=38.8GiB | 49.3% done | ETA(epoch): 499s + [Epoch 246][50/66] loss=0.930053 avg=0.849907 VRAM=39.0GiB | 49.4% done | ETA(epoch): 308s + [Epoch 246][60/66] loss=0.870583 avg=0.853060 VRAM=38.8GiB | 49.4% done | ETA(epoch): 115s + Train loss: 0.851482 (1269.4s) ETA: 5831min + Val loss: 0.822305 [t_0.0-0.2=1.0720 t_0.2-0.4=1.0085 t_0.4-0.6=0.8384 t_0.6-0.8=0.7228 t_0.8-1.0=0.6196] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 247/499 (49% done) --- + [Epoch 247][10/66] loss=0.782461 avg=0.888502 VRAM=39.0GiB | 49.4% done | ETA(epoch): 1073s + [Epoch 247][20/66] loss=0.924429 avg=0.891660 VRAM=38.8GiB | 49.5% done | ETA(epoch): 882s + [Epoch 247][30/66] loss=0.661241 avg=0.867161 VRAM=39.0GiB | 49.5% done | ETA(epoch): 691s + [Epoch 247][40/66] loss=0.712082 avg=0.849480 VRAM=38.8GiB | 49.5% done | ETA(epoch): 499s + [Epoch 247][50/66] loss=0.876890 avg=0.848686 VRAM=39.0GiB | 49.6% done | ETA(epoch): 307s + [Epoch 247][60/66] loss=1.022200 avg=0.852682 VRAM=38.8GiB | 49.6% done | ETA(epoch): 115s + Train loss: 0.856760 (1267.0s) ETA: 5808min + Val loss: 0.871725 [t_0.0-0.2=1.0920 t_0.2-0.4=0.9798 t_0.4-0.6=0.8113 t_0.6-0.8=0.7115 t_0.8-1.0=0.6253] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 248/499 (50% done) --- + [Epoch 248][10/66] loss=0.796491 avg=0.841349 VRAM=39.0GiB | 49.6% done | ETA(epoch): 1075s + [Epoch 248][20/66] loss=0.869188 avg=0.861702 VRAM=38.8GiB | 49.7% done | ETA(epoch): 882s + [Epoch 248][30/66] loss=0.848295 avg=0.850519 VRAM=39.0GiB | 49.7% done | ETA(epoch): 696s + [Epoch 248][40/66] loss=0.747250 avg=0.850303 VRAM=38.8GiB | 49.7% done | ETA(epoch): 501s + [Epoch 248][50/66] loss=0.865552 avg=0.849822 VRAM=39.0GiB | 49.8% done | ETA(epoch): 308s + [Epoch 248][60/66] loss=0.928061 avg=0.848777 VRAM=38.8GiB | 49.8% done | ETA(epoch): 116s + Train loss: 0.852157 (1270.2s) ETA: 5785min + Val loss: 0.858591 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9933 t_0.4-0.6=0.8502 t_0.6-0.8=0.7049 t_0.8-1.0=0.6267] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0248 +[MEM @ epoch 248 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 249/499 (50% done) --- + [Epoch 249][10/66] loss=0.898284 avg=0.823215 VRAM=39.0GiB | 49.8% done | ETA(epoch): 1077s + [Epoch 249][20/66] loss=0.824297 avg=0.835396 VRAM=38.8GiB | 49.9% done | ETA(epoch): 886s + [Epoch 249][30/66] loss=0.845198 avg=0.850413 VRAM=39.0GiB | 49.9% done | ETA(epoch): 693s + [Epoch 249][40/66] loss=0.782714 avg=0.846988 VRAM=38.8GiB | 49.9% done | ETA(epoch): 500s + [Epoch 249][50/66] loss=0.878662 avg=0.853208 VRAM=39.0GiB | 50.0% done | ETA(epoch): 308s + [Epoch 249][60/66] loss=0.846749 avg=0.851289 VRAM=38.8GiB | 50.0% done | ETA(epoch): 115s + Train loss: 0.853353 (1268.2s) ETA: 5761min + Val loss: 0.848332 [t_0.0-0.2=1.0695 t_0.2-0.4=1.0111 t_0.4-0.6=0.8788 t_0.6-0.8=0.6908 t_0.8-1.0=0.6225] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 250/499 (50% done) --- + [MilestoneVis] train_0 step 16500 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 16500 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 16500 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 16500 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 16500 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 16500 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 16500 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 16500 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 16500 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 16500 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 250][10/66] loss=0.864252 avg=0.818020 VRAM=39.0GiB | 50.0% done | ETA(epoch): 6884s + [Epoch 250][20/66] loss=0.944225 avg=0.837260 VRAM=38.8GiB | 50.1% done | ETA(epoch): 3269s + [Epoch 250][30/66] loss=0.957342 avg=0.847060 VRAM=39.0GiB | 50.1% done | ETA(epoch): 1936s + [Epoch 250][40/66] loss=0.834694 avg=0.842924 VRAM=38.8GiB | 50.1% done | ETA(epoch): 1173s + [Epoch 250][50/66] loss=0.850240 avg=0.839924 VRAM=39.0GiB | 50.2% done | ETA(epoch): 639s + [Epoch 250][60/66] loss=0.919451 avg=0.846148 VRAM=38.8GiB | 50.2% done | ETA(epoch): 219s + Train loss: 0.844270 (2303.9s) ETA: 5755min + Val loss: 0.843244 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9465 t_0.4-0.6=0.8421 t_0.6-0.8=0.7225 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 251/499 (50% done) --- + [Epoch 251][10/66] loss=0.881935 avg=0.805914 VRAM=39.0GiB | 50.2% done | ETA(epoch): 1072s + [Epoch 251][20/66] loss=0.776765 avg=0.836966 VRAM=38.8GiB | 50.3% done | ETA(epoch): 886s + [Epoch 251][30/66] loss=0.916684 avg=0.836244 VRAM=39.0GiB | 50.3% done | ETA(epoch): 693s + [Epoch 251][40/66] loss=0.749854 avg=0.834086 VRAM=38.8GiB | 50.3% done | ETA(epoch): 500s + [Epoch 251][50/66] loss=0.758061 avg=0.840357 VRAM=39.0GiB | 50.4% done | ETA(epoch): 308s + [Epoch 251][60/66] loss=0.932120 avg=0.843326 VRAM=38.8GiB | 50.4% done | ETA(epoch): 115s + Train loss: 0.851906 (1268.5s) ETA: 5732min + Val loss: 0.871665 [t_0.0-0.2=1.0668 t_0.2-0.4=1.0030 t_0.4-0.6=0.8310 t_0.6-0.8=0.6937 t_0.8-1.0=0.6048] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 252/499 (50% done) --- + [Epoch 252][10/66] loss=0.864867 avg=0.827380 VRAM=39.0GiB | 50.4% done | ETA(epoch): 1075s + [Epoch 252][20/66] loss=0.746982 avg=0.851564 VRAM=38.8GiB | 50.5% done | ETA(epoch): 883s + [Epoch 252][30/66] loss=0.813190 avg=0.853979 VRAM=39.0GiB | 50.5% done | ETA(epoch): 691s + [Epoch 252][40/66] loss=0.897701 avg=0.854424 VRAM=38.8GiB | 50.5% done | ETA(epoch): 499s + [Epoch 252][50/66] loss=0.867198 avg=0.853151 VRAM=39.0GiB | 50.6% done | ETA(epoch): 307s + [Epoch 252][60/66] loss=0.883770 avg=0.853112 VRAM=38.8GiB | 50.6% done | ETA(epoch): 115s + Train loss: 0.850166 (1265.8s) ETA: 5708min + Val loss: 0.846224 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9629 t_0.4-0.6=0.8276 t_0.6-0.8=0.7086 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 253/499 (51% done) --- + [Epoch 253][10/66] loss=0.957408 avg=0.868170 VRAM=39.0GiB | 50.6% done | ETA(epoch): 1074s + [Epoch 253][20/66] loss=0.782567 avg=0.856744 VRAM=38.8GiB | 50.7% done | ETA(epoch): 883s + [Epoch 253][30/66] loss=0.817081 avg=0.858932 VRAM=39.0GiB | 50.7% done | ETA(epoch): 691s + [Epoch 253][40/66] loss=0.887505 avg=0.848564 VRAM=38.8GiB | 50.7% done | ETA(epoch): 499s + [Epoch 253][50/66] loss=0.819125 avg=0.847908 VRAM=39.0GiB | 50.8% done | ETA(epoch): 307s + [Epoch 253][60/66] loss=0.742782 avg=0.845567 VRAM=38.8GiB | 50.8% done | ETA(epoch): 115s + Train loss: 0.848333 (1266.2s) ETA: 5685min + Val loss: 0.889452 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9930 t_0.4-0.6=0.8307 t_0.6-0.8=0.7169 t_0.8-1.0=0.6210] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 254/499 (51% done) --- + [Epoch 254][10/66] loss=0.867179 avg=0.856170 VRAM=39.0GiB | 50.8% done | ETA(epoch): 1073s + [Epoch 254][20/66] loss=0.792731 avg=0.838287 VRAM=38.8GiB | 50.9% done | ETA(epoch): 882s + [Epoch 254][30/66] loss=0.845960 avg=0.837158 VRAM=39.0GiB | 50.9% done | ETA(epoch): 690s + [Epoch 254][40/66] loss=0.868967 avg=0.848616 VRAM=38.8GiB | 50.9% done | ETA(epoch): 498s + [Epoch 254][50/66] loss=0.914970 avg=0.854178 VRAM=39.0GiB | 51.0% done | ETA(epoch): 307s + [Epoch 254][60/66] loss=0.902225 avg=0.858247 VRAM=38.8GiB | 51.0% done | ETA(epoch): 115s + Train loss: 0.858285 (1267.2s) ETA: 5661min + Val loss: 0.857459 [t_0.0-0.2=1.0717 t_0.2-0.4=0.9912 t_0.4-0.6=0.8706 t_0.6-0.8=0.7039 t_0.8-1.0=0.6242] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 255/499 (51% done) --- + [Epoch 255][10/66] loss=0.767201 avg=0.829020 VRAM=39.0GiB | 51.0% done | ETA(epoch): 1073s + [Epoch 255][20/66] loss=0.832656 avg=0.839118 VRAM=38.8GiB | 51.1% done | ETA(epoch): 882s + [Epoch 255][30/66] loss=0.748953 avg=0.832825 VRAM=39.0GiB | 51.1% done | ETA(epoch): 690s + [Epoch 255][40/66] loss=1.050344 avg=0.844688 VRAM=38.8GiB | 51.1% done | ETA(epoch): 498s + [Epoch 255][50/66] loss=0.802449 avg=0.845839 VRAM=39.0GiB | 51.2% done | ETA(epoch): 307s + [Epoch 255][60/66] loss=0.864515 avg=0.848519 VRAM=38.8GiB | 51.2% done | ETA(epoch): 115s + Train loss: 0.847786 (1265.2s) ETA: 5638min + Val loss: 0.839462 [t_0.0-0.2=1.0567 t_0.2-0.4=0.9570 t_0.4-0.6=0.8595 t_0.6-0.8=0.7179 t_0.8-1.0=0.6212] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 256/499 (51% done) --- + [Epoch 256][10/66] loss=0.904764 avg=0.864215 VRAM=39.0GiB | 51.2% done | ETA(epoch): 1073s + [Epoch 256][20/66] loss=0.805431 avg=0.851509 VRAM=38.8GiB | 51.3% done | ETA(epoch): 883s + [Epoch 256][30/66] loss=0.697580 avg=0.839344 VRAM=39.0GiB | 51.3% done | ETA(epoch): 691s + [Epoch 256][40/66] loss=0.919018 avg=0.849961 VRAM=38.8GiB | 51.3% done | ETA(epoch): 500s + [Epoch 256][50/66] loss=0.885374 avg=0.847517 VRAM=39.0GiB | 51.4% done | ETA(epoch): 307s + [Epoch 256][60/66] loss=0.868051 avg=0.857840 VRAM=38.8GiB | 51.4% done | ETA(epoch): 115s + Train loss: 0.863475 (1267.8s) ETA: 5614min + Val loss: 0.851587 [t_0.0-0.2=1.0597 t_0.2-0.4=0.9904 t_0.4-0.6=0.8535 t_0.6-0.8=0.7045 t_0.8-1.0=0.6251] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 257/499 (51% done) --- + [Epoch 257][10/66] loss=0.904388 avg=0.850830 VRAM=39.0GiB | 51.4% done | ETA(epoch): 1073s + [Epoch 257][20/66] loss=0.906177 avg=0.845816 VRAM=38.8GiB | 51.5% done | ETA(epoch): 882s + [Epoch 257][30/66] loss=0.846103 avg=0.851198 VRAM=39.0GiB | 51.5% done | ETA(epoch): 696s + [Epoch 257][40/66] loss=0.813856 avg=0.847353 VRAM=38.8GiB | 51.5% done | ETA(epoch): 502s + [Epoch 257][50/66] loss=0.901769 avg=0.848589 VRAM=39.0GiB | 51.6% done | ETA(epoch): 308s + [Epoch 257][60/66] loss=0.900772 avg=0.853285 VRAM=38.8GiB | 51.6% done | ETA(epoch): 116s + Train loss: 0.850686 (1270.6s) ETA: 5591min + Val loss: 0.871870 [t_0.0-0.2=1.0738 t_0.2-0.4=1.0162 t_0.4-0.6=0.8437 t_0.6-0.8=0.7081 t_0.8-1.0=0.6104] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 258/499 (52% done) --- + [Epoch 258][10/66] loss=1.010759 avg=0.883119 VRAM=39.0GiB | 51.6% done | ETA(epoch): 1073s + [Epoch 258][20/66] loss=0.908710 avg=0.869924 VRAM=38.8GiB | 51.7% done | ETA(epoch): 882s + [Epoch 258][30/66] loss=0.893336 avg=0.858830 VRAM=39.0GiB | 51.7% done | ETA(epoch): 690s + [Epoch 258][40/66] loss=0.964494 avg=0.864027 VRAM=38.8GiB | 51.7% done | ETA(epoch): 499s + [Epoch 258][50/66] loss=0.720256 avg=0.859133 VRAM=39.0GiB | 51.8% done | ETA(epoch): 307s + [Epoch 258][60/66] loss=0.758329 avg=0.851615 VRAM=38.8GiB | 51.8% done | ETA(epoch): 115s + Train loss: 0.851165 (1265.6s) ETA: 5567min + Val loss: 0.839032 [t_0.0-0.2=1.0664 t_0.2-0.4=1.0048 t_0.4-0.6=0.8304 t_0.6-0.8=0.6922 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 259/499 (52% done) --- + [Epoch 259][10/66] loss=0.944494 avg=0.793757 VRAM=39.0GiB | 51.8% done | ETA(epoch): 1073s + [Epoch 259][20/66] loss=0.899853 avg=0.835058 VRAM=38.8GiB | 51.9% done | ETA(epoch): 882s + [Epoch 259][30/66] loss=0.702512 avg=0.833891 VRAM=39.0GiB | 51.9% done | ETA(epoch): 690s + [Epoch 259][40/66] loss=0.955138 avg=0.831712 VRAM=38.8GiB | 51.9% done | ETA(epoch): 499s + [Epoch 259][50/66] loss=0.872873 avg=0.838473 VRAM=39.0GiB | 52.0% done | ETA(epoch): 307s + [Epoch 259][60/66] loss=0.866873 avg=0.846108 VRAM=38.8GiB | 52.0% done | ETA(epoch): 115s + Train loss: 0.847026 (1266.0s) ETA: 5544min + Val loss: 0.848149 [t_0.0-0.2=1.0687 t_0.2-0.4=1.0023 t_0.4-0.6=0.8386 t_0.6-0.8=0.6935 t_0.8-1.0=0.6233] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 260/499 (52% done) --- + [Epoch 260][10/66] loss=0.785915 avg=0.831694 VRAM=39.0GiB | 52.0% done | ETA(epoch): 1086s + [Epoch 260][20/66] loss=0.702906 avg=0.833078 VRAM=38.8GiB | 52.1% done | ETA(epoch): 888s + [Epoch 260][30/66] loss=0.774945 avg=0.836998 VRAM=39.0GiB | 52.1% done | ETA(epoch): 693s + [Epoch 260][40/66] loss=0.668392 avg=0.833980 VRAM=38.8GiB | 52.1% done | ETA(epoch): 500s + [Epoch 260][50/66] loss=0.871855 avg=0.848022 VRAM=39.0GiB | 52.2% done | ETA(epoch): 308s + [Epoch 260][60/66] loss=0.732682 avg=0.849530 VRAM=38.8GiB | 52.2% done | ETA(epoch): 115s + Train loss: 0.847457 (1273.4s) ETA: 5520min + Val loss: 0.860671 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0108 t_0.4-0.6=0.8426 t_0.6-0.8=0.7052 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 261/499 (52% done) --- + [Epoch 261][10/66] loss=0.857832 avg=0.855307 VRAM=39.0GiB | 52.2% done | ETA(epoch): 1073s + [Epoch 261][20/66] loss=0.768152 avg=0.843924 VRAM=38.8GiB | 52.3% done | ETA(epoch): 881s + [Epoch 261][30/66] loss=0.864206 avg=0.845457 VRAM=39.0GiB | 52.3% done | ETA(epoch): 691s + [Epoch 261][40/66] loss=0.811137 avg=0.833822 VRAM=38.8GiB | 52.3% done | ETA(epoch): 499s + [Epoch 261][50/66] loss=0.806875 avg=0.841422 VRAM=39.0GiB | 52.4% done | ETA(epoch): 307s + [Epoch 261][60/66] loss=0.942151 avg=0.848235 VRAM=38.8GiB | 52.4% done | ETA(epoch): 115s + Train loss: 0.858499 (1266.3s) ETA: 5497min + Val loss: 0.838052 [t_0.0-0.2=1.0556 t_0.2-0.4=1.0193 t_0.4-0.6=0.8396 t_0.6-0.8=0.7031 t_0.8-1.0=0.6215] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 262/499 (52% done) --- + [Epoch 262][10/66] loss=1.008481 avg=0.848908 VRAM=39.0GiB | 52.4% done | ETA(epoch): 1073s + [Epoch 262][20/66] loss=0.711175 avg=0.841484 VRAM=38.8GiB | 52.5% done | ETA(epoch): 882s + [Epoch 262][30/66] loss=0.898633 avg=0.854068 VRAM=39.0GiB | 52.5% done | ETA(epoch): 691s + [Epoch 262][40/66] loss=0.700176 avg=0.850872 VRAM=38.8GiB | 52.5% done | ETA(epoch): 499s + [Epoch 262][50/66] loss=0.809977 avg=0.849204 VRAM=39.0GiB | 52.6% done | ETA(epoch): 308s + [Epoch 262][60/66] loss=0.912768 avg=0.849099 VRAM=38.8GiB | 52.6% done | ETA(epoch): 115s + Train loss: 0.842904 (1269.1s) ETA: 5473min + Val loss: 0.858293 [t_0.0-0.2=1.0647 t_0.2-0.4=0.9785 t_0.4-0.6=0.8311 t_0.6-0.8=0.7064 t_0.8-1.0=0.6042] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 263/499 (53% done) --- + [Epoch 263][10/66] loss=0.996481 avg=0.853974 VRAM=39.0GiB | 52.6% done | ETA(epoch): 1075s + [Epoch 263][20/66] loss=0.859350 avg=0.840405 VRAM=38.8GiB | 52.7% done | ETA(epoch): 882s + [Epoch 263][30/66] loss=0.794792 avg=0.837363 VRAM=39.0GiB | 52.7% done | ETA(epoch): 691s + [Epoch 263][40/66] loss=0.917707 avg=0.836764 VRAM=38.8GiB | 52.7% done | ETA(epoch): 499s + [Epoch 263][50/66] loss=0.761122 avg=0.841056 VRAM=39.0GiB | 52.8% done | ETA(epoch): 307s + [Epoch 263][60/66] loss=0.789156 avg=0.838598 VRAM=38.8GiB | 52.8% done | ETA(epoch): 115s + Train loss: 0.841069 (1266.8s) ETA: 5450min + Val loss: 0.872621 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9897 t_0.4-0.6=0.8694 t_0.6-0.8=0.6954 t_0.8-1.0=0.6206] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 264/499 (53% done) --- + [Epoch 264][10/66] loss=0.882815 avg=0.822704 VRAM=39.0GiB | 52.8% done | ETA(epoch): 1074s + [Epoch 264][20/66] loss=0.839291 avg=0.834633 VRAM=38.8GiB | 52.9% done | ETA(epoch): 883s + [Epoch 264][30/66] loss=0.756092 avg=0.833120 VRAM=39.0GiB | 52.9% done | ETA(epoch): 691s + [Epoch 264][40/66] loss=0.740450 avg=0.828183 VRAM=38.8GiB | 52.9% done | ETA(epoch): 499s + [Epoch 264][50/66] loss=0.813732 avg=0.829158 VRAM=39.0GiB | 53.0% done | ETA(epoch): 307s + [Epoch 264][60/66] loss=0.826727 avg=0.830124 VRAM=38.8GiB | 53.0% done | ETA(epoch): 115s + Train loss: 0.834477 (1266.9s) ETA: 5426min + Val loss: 0.872468 [t_0.0-0.2=1.0580 t_0.2-0.4=1.0139 t_0.4-0.6=0.8755 t_0.6-0.8=0.7179 t_0.8-1.0=0.6120] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 265/499 (53% done) --- + [Epoch 265][10/66] loss=1.035925 avg=0.909774 VRAM=39.0GiB | 53.0% done | ETA(epoch): 1073s + [Epoch 265][20/66] loss=0.915499 avg=0.871237 VRAM=38.8GiB | 53.1% done | ETA(epoch): 881s + [Epoch 265][30/66] loss=0.837631 avg=0.862852 VRAM=39.0GiB | 53.1% done | ETA(epoch): 693s + [Epoch 265][40/66] loss=0.817092 avg=0.861151 VRAM=38.8GiB | 53.1% done | ETA(epoch): 500s + [Epoch 265][50/66] loss=0.689685 avg=0.856298 VRAM=39.0GiB | 53.2% done | ETA(epoch): 307s + [Epoch 265][60/66] loss=0.922758 avg=0.852792 VRAM=38.8GiB | 53.2% done | ETA(epoch): 115s + Train loss: 0.856789 (1267.7s) ETA: 5403min + Val loss: 0.833591 [t_0.0-0.2=1.0914 t_0.2-0.4=0.9832 t_0.4-0.6=0.8611 t_0.6-0.8=0.6837 t_0.8-1.0=0.6257] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 266/499 (53% done) --- + [Epoch 266][10/66] loss=0.909781 avg=0.833391 VRAM=39.0GiB | 53.2% done | ETA(epoch): 1075s + [Epoch 266][20/66] loss=0.894944 avg=0.861812 VRAM=38.8GiB | 53.3% done | ETA(epoch): 882s + [Epoch 266][30/66] loss=0.862628 avg=0.855790 VRAM=39.0GiB | 53.3% done | ETA(epoch): 691s + [Epoch 266][40/66] loss=0.793263 avg=0.849497 VRAM=38.8GiB | 53.3% done | ETA(epoch): 499s + [Epoch 266][50/66] loss=0.764562 avg=0.847703 VRAM=39.0GiB | 53.4% done | ETA(epoch): 307s + [Epoch 266][60/66] loss=0.724138 avg=0.842814 VRAM=38.8GiB | 53.4% done | ETA(epoch): 115s + Train loss: 0.844867 (1266.8s) ETA: 5380min + Val loss: 0.839074 [t_0.0-0.2=1.0658 t_0.2-0.4=0.9834 t_0.4-0.6=0.8505 t_0.6-0.8=0.7272 t_0.8-1.0=0.6340] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 267/499 (53% done) --- + [Epoch 267][10/66] loss=0.743314 avg=0.805428 VRAM=39.0GiB | 53.4% done | ETA(epoch): 1073s + [Epoch 267][20/66] loss=0.867910 avg=0.844728 VRAM=38.8GiB | 53.5% done | ETA(epoch): 882s + [Epoch 267][30/66] loss=0.966607 avg=0.850964 VRAM=39.0GiB | 53.5% done | ETA(epoch): 690s + [Epoch 267][40/66] loss=0.840813 avg=0.861178 VRAM=38.8GiB | 53.5% done | ETA(epoch): 498s + [Epoch 267][50/66] loss=0.767989 avg=0.856455 VRAM=39.0GiB | 53.6% done | ETA(epoch): 307s + [Epoch 267][60/66] loss=0.843731 avg=0.855434 VRAM=38.8GiB | 53.6% done | ETA(epoch): 115s + Train loss: 0.856818 (1265.2s) ETA: 5356min + Val loss: 0.832399 [t_0.0-0.2=1.0644 t_0.2-0.4=0.9831 t_0.4-0.6=0.8274 t_0.6-0.8=0.6908 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 268/499 (54% done) --- + [Epoch 268][10/66] loss=0.723229 avg=0.856929 VRAM=39.0GiB | 53.6% done | ETA(epoch): 1074s + [Epoch 268][20/66] loss=0.975811 avg=0.870483 VRAM=38.8GiB | 53.7% done | ETA(epoch): 882s + [Epoch 268][30/66] loss=0.842184 avg=0.854300 VRAM=39.0GiB | 53.7% done | ETA(epoch): 691s + [Epoch 268][40/66] loss=0.848777 avg=0.848973 VRAM=38.8GiB | 53.7% done | ETA(epoch): 499s + [Epoch 268][50/66] loss=0.827576 avg=0.851354 VRAM=39.0GiB | 53.8% done | ETA(epoch): 307s + [Epoch 268][60/66] loss=0.822335 avg=0.844595 VRAM=38.8GiB | 53.8% done | ETA(epoch): 115s + Train loss: 0.842788 (1266.6s) ETA: 5333min + Val loss: 0.854342 [t_0.0-0.2=1.0488 t_0.2-0.4=1.0301 t_0.4-0.6=0.8435 t_0.6-0.8=0.7071 t_0.8-1.0=0.6060] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 269/499 (54% done) --- + [Epoch 269][10/66] loss=0.828719 avg=0.821689 VRAM=39.0GiB | 53.8% done | ETA(epoch): 1074s + [Epoch 269][20/66] loss=0.939868 avg=0.842050 VRAM=38.8GiB | 53.9% done | ETA(epoch): 883s + [Epoch 269][30/66] loss=0.899383 avg=0.844804 VRAM=39.0GiB | 53.9% done | ETA(epoch): 691s + [Epoch 269][40/66] loss=0.929070 avg=0.851940 VRAM=38.8GiB | 53.9% done | ETA(epoch): 499s + [Epoch 269][50/66] loss=0.860231 avg=0.860103 VRAM=39.0GiB | 54.0% done | ETA(epoch): 307s + [Epoch 269][60/66] loss=0.891953 avg=0.861849 VRAM=38.8GiB | 54.0% done | ETA(epoch): 115s + Train loss: 0.862814 (1268.5s) ETA: 5309min + Val loss: 0.869865 [t_0.0-0.2=1.0637 t_0.2-0.4=0.9739 t_0.4-0.6=0.8728 t_0.6-0.8=0.7123 t_0.8-1.0=0.6219] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 270/499 (54% done) --- + [Epoch 270][10/66] loss=0.918824 avg=0.854038 VRAM=39.0GiB | 54.0% done | ETA(epoch): 1074s + [Epoch 270][20/66] loss=0.854714 avg=0.846645 VRAM=38.8GiB | 54.1% done | ETA(epoch): 883s + [Epoch 270][30/66] loss=0.828381 avg=0.837866 VRAM=39.0GiB | 54.1% done | ETA(epoch): 691s + [Epoch 270][40/66] loss=0.882509 avg=0.848816 VRAM=38.8GiB | 54.1% done | ETA(epoch): 499s + [Epoch 270][50/66] loss=0.875494 avg=0.842376 VRAM=39.0GiB | 54.2% done | ETA(epoch): 307s + [Epoch 270][60/66] loss=0.866468 avg=0.843507 VRAM=38.8GiB | 54.2% done | ETA(epoch): 115s + Train loss: 0.842835 (1267.0s) ETA: 5286min + Val loss: 0.853163 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9714 t_0.4-0.6=0.8513 t_0.6-0.8=0.7200 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 271/499 (54% done) --- + [Epoch 271][10/66] loss=0.854010 avg=0.875873 VRAM=39.0GiB | 54.2% done | ETA(epoch): 1073s + [Epoch 271][20/66] loss=0.871254 avg=0.864033 VRAM=38.8GiB | 54.3% done | ETA(epoch): 883s + [Epoch 271][30/66] loss=0.765571 avg=0.851033 VRAM=39.0GiB | 54.3% done | ETA(epoch): 691s + [Epoch 271][40/66] loss=0.933117 avg=0.858699 VRAM=38.8GiB | 54.3% done | ETA(epoch): 501s + [Epoch 271][50/66] loss=0.865096 avg=0.846939 VRAM=39.0GiB | 54.4% done | ETA(epoch): 308s + [Epoch 271][60/66] loss=0.853515 avg=0.840798 VRAM=38.8GiB | 54.4% done | ETA(epoch): 115s + Train loss: 0.842668 (1269.4s) ETA: 5263min + Val loss: 0.859993 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0320 t_0.4-0.6=0.8488 t_0.6-0.8=0.6870 t_0.8-1.0=0.6242] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 272/499 (54% done) --- + [Epoch 272][10/66] loss=0.841795 avg=0.815418 VRAM=39.0GiB | 54.4% done | ETA(epoch): 1073s + [Epoch 272][20/66] loss=0.728800 avg=0.831161 VRAM=38.8GiB | 54.5% done | ETA(epoch): 882s + [Epoch 272][30/66] loss=0.784847 avg=0.839458 VRAM=39.0GiB | 54.5% done | ETA(epoch): 694s + [Epoch 272][40/66] loss=0.838795 avg=0.829998 VRAM=38.8GiB | 54.5% done | ETA(epoch): 501s + [Epoch 272][50/66] loss=0.883726 avg=0.835426 VRAM=39.0GiB | 54.6% done | ETA(epoch): 308s + [Epoch 272][60/66] loss=0.936443 avg=0.840809 VRAM=38.8GiB | 54.6% done | ETA(epoch): 115s + Train loss: 0.840706 (1269.0s) ETA: 5239min + Val loss: 0.862659 [t_0.0-0.2=1.0658 t_0.2-0.4=1.0447 t_0.4-0.6=0.8513 t_0.6-0.8=0.7041 t_0.8-1.0=0.6107] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 273/499 (55% done) --- + [Epoch 273][10/66] loss=0.846858 avg=0.862414 VRAM=39.0GiB | 54.6% done | ETA(epoch): 1075s + [Epoch 273][20/66] loss=0.869297 avg=0.854032 VRAM=38.8GiB | 54.7% done | ETA(epoch): 883s + [Epoch 273][30/66] loss=0.768364 avg=0.858960 VRAM=39.0GiB | 54.7% done | ETA(epoch): 691s + [Epoch 273][40/66] loss=0.813122 avg=0.862836 VRAM=38.8GiB | 54.7% done | ETA(epoch): 499s + [Epoch 273][50/66] loss=0.801949 avg=0.858593 VRAM=39.0GiB | 54.8% done | ETA(epoch): 307s + [Epoch 273][60/66] loss=0.929245 avg=0.855429 VRAM=38.8GiB | 54.8% done | ETA(epoch): 115s + Train loss: 0.852659 (1266.5s) ETA: 5216min + Val loss: 0.837336 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9895 t_0.4-0.6=0.8068 t_0.6-0.8=0.6983 t_0.8-1.0=0.6136] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 274/499 (55% done) --- + [Epoch 274][10/66] loss=0.808278 avg=0.842693 VRAM=39.0GiB | 54.8% done | ETA(epoch): 1073s + [Epoch 274][20/66] loss=0.938031 avg=0.831645 VRAM=38.8GiB | 54.9% done | ETA(epoch): 886s + [Epoch 274][30/66] loss=0.841562 avg=0.829817 VRAM=39.0GiB | 54.9% done | ETA(epoch): 693s + [Epoch 274][40/66] loss=0.898898 avg=0.830328 VRAM=38.8GiB | 54.9% done | ETA(epoch): 500s + [Epoch 274][50/66] loss=0.666932 avg=0.832433 VRAM=39.0GiB | 55.0% done | ETA(epoch): 308s + [Epoch 274][60/66] loss=0.926055 avg=0.831751 VRAM=38.8GiB | 55.0% done | ETA(epoch): 115s + Train loss: 0.836272 (1269.6s) ETA: 5192min + Val loss: 0.823412 [t_0.0-0.2=1.0337 t_0.2-0.4=0.9918 t_0.4-0.6=0.8205 t_0.6-0.8=0.6963 t_0.8-1.0=0.6339] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 275/499 (55% done) --- + [Epoch 275][10/66] loss=0.889780 avg=0.824392 VRAM=39.0GiB | 55.0% done | ETA(epoch): 1074s + [Epoch 275][20/66] loss=0.902579 avg=0.846559 VRAM=38.8GiB | 55.1% done | ETA(epoch): 883s + [Epoch 275][30/66] loss=0.879671 avg=0.832600 VRAM=39.0GiB | 55.1% done | ETA(epoch): 691s + [Epoch 275][40/66] loss=0.741690 avg=0.832232 VRAM=38.8GiB | 55.1% done | ETA(epoch): 499s + [Epoch 275][50/66] loss=0.791134 avg=0.830704 VRAM=39.0GiB | 55.2% done | ETA(epoch): 307s + [Epoch 275][60/66] loss=0.808895 avg=0.831642 VRAM=38.8GiB | 55.2% done | ETA(epoch): 115s + Train loss: 0.830261 (1266.4s) ETA: 5169min + Val loss: 0.833374 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9761 t_0.4-0.6=0.8485 t_0.6-0.8=0.7243 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 276/499 (55% done) --- + [Epoch 276][10/66] loss=0.875876 avg=0.863407 VRAM=39.0GiB | 55.2% done | ETA(epoch): 1077s + [Epoch 276][20/66] loss=0.765768 avg=0.854450 VRAM=38.8GiB | 55.3% done | ETA(epoch): 883s + [Epoch 276][30/66] loss=0.853662 avg=0.854969 VRAM=39.0GiB | 55.3% done | ETA(epoch): 691s + [Epoch 276][40/66] loss=0.649002 avg=0.831370 VRAM=38.8GiB | 55.3% done | ETA(epoch): 499s + [Epoch 276][50/66] loss=0.823043 avg=0.842056 VRAM=39.0GiB | 55.4% done | ETA(epoch): 307s + [Epoch 276][60/66] loss=0.741444 avg=0.837406 VRAM=38.8GiB | 55.4% done | ETA(epoch): 115s + Train loss: 0.836495 (1266.9s) ETA: 5146min + Val loss: 0.810512 [t_0.0-0.2=1.0555 t_0.2-0.4=1.0153 t_0.4-0.6=0.8278 t_0.6-0.8=0.6895 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 277/499 (55% done) --- + [Epoch 277][10/66] loss=0.823512 avg=0.819861 VRAM=39.0GiB | 55.4% done | ETA(epoch): 1075s + [Epoch 277][20/66] loss=0.796900 avg=0.820556 VRAM=38.8GiB | 55.5% done | ETA(epoch): 882s + [Epoch 277][30/66] loss=0.889054 avg=0.824867 VRAM=39.0GiB | 55.5% done | ETA(epoch): 691s + [Epoch 277][40/66] loss=0.833393 avg=0.832656 VRAM=38.8GiB | 55.5% done | ETA(epoch): 499s + [Epoch 277][50/66] loss=0.803247 avg=0.832335 VRAM=39.0GiB | 55.6% done | ETA(epoch): 307s + [Epoch 277][60/66] loss=0.725319 avg=0.830982 VRAM=38.8GiB | 55.6% done | ETA(epoch): 115s + Train loss: 0.828368 (1266.8s) ETA: 5122min + Val loss: 0.870170 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9840 t_0.4-0.6=0.8479 t_0.6-0.8=0.7170 t_0.8-1.0=0.6140] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 278/499 (56% done) --- + [Epoch 278][10/66] loss=0.909280 avg=0.891844 VRAM=39.0GiB | 55.6% done | ETA(epoch): 1075s + [Epoch 278][20/66] loss=0.795791 avg=0.879614 VRAM=38.8GiB | 55.7% done | ETA(epoch): 883s + [Epoch 278][30/66] loss=0.973380 avg=0.874197 VRAM=39.0GiB | 55.7% done | ETA(epoch): 691s + [Epoch 278][40/66] loss=0.813103 avg=0.873110 VRAM=38.8GiB | 55.7% done | ETA(epoch): 499s + [Epoch 278][50/66] loss=0.886491 avg=0.871053 VRAM=39.0GiB | 55.8% done | ETA(epoch): 308s + [Epoch 278][60/66] loss=0.857430 avg=0.861311 VRAM=38.8GiB | 55.8% done | ETA(epoch): 115s + Train loss: 0.861615 (1268.9s) ETA: 5099min + Val loss: 0.827376 [t_0.0-0.2=1.0415 t_0.2-0.4=1.0051 t_0.4-0.6=0.8281 t_0.6-0.8=0.7057 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 279/499 (56% done) --- + [Epoch 279][10/66] loss=0.892640 avg=0.844265 VRAM=39.0GiB | 55.8% done | ETA(epoch): 1072s + [Epoch 279][20/66] loss=0.771943 avg=0.844415 VRAM=38.8GiB | 55.9% done | ETA(epoch): 880s + [Epoch 279][30/66] loss=0.841565 avg=0.831147 VRAM=39.0GiB | 55.9% done | ETA(epoch): 689s + [Epoch 279][40/66] loss=0.999242 avg=0.835730 VRAM=38.8GiB | 55.9% done | ETA(epoch): 498s + [Epoch 279][50/66] loss=0.846740 avg=0.845576 VRAM=39.0GiB | 56.0% done | ETA(epoch): 307s + [Epoch 279][60/66] loss=0.854945 avg=0.847269 VRAM=38.8GiB | 56.0% done | ETA(epoch): 115s + Train loss: 0.849833 (1265.7s) ETA: 5076min + Val loss: 0.871992 [t_0.0-0.2=1.0801 t_0.2-0.4=1.0037 t_0.4-0.6=0.8679 t_0.6-0.8=0.7176 t_0.8-1.0=0.6163] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 280/499 (56% done) --- + [Epoch 280][10/66] loss=0.928696 avg=0.869390 VRAM=39.0GiB | 56.0% done | ETA(epoch): 1074s + [Epoch 280][20/66] loss=0.834834 avg=0.855198 VRAM=38.8GiB | 56.1% done | ETA(epoch): 884s + [Epoch 280][30/66] loss=0.926616 avg=0.859290 VRAM=39.0GiB | 56.1% done | ETA(epoch): 698s + [Epoch 280][40/66] loss=0.919106 avg=0.854198 VRAM=38.8GiB | 56.1% done | ETA(epoch): 503s + [Epoch 280][50/66] loss=0.778843 avg=0.856493 VRAM=39.0GiB | 56.2% done | ETA(epoch): 309s + [Epoch 280][60/66] loss=0.848006 avg=0.855780 VRAM=38.8GiB | 56.2% done | ETA(epoch): 116s + Train loss: 0.853262 (1272.9s) ETA: 5052min + Val loss: 0.821158 [t_0.0-0.2=1.0655 t_0.2-0.4=0.9745 t_0.4-0.6=0.8278 t_0.6-0.8=0.7025 t_0.8-1.0=0.6296] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 281/499 (56% done) --- + [Epoch 281][10/66] loss=0.764130 avg=0.825620 VRAM=39.0GiB | 56.2% done | ETA(epoch): 1075s + [Epoch 281][20/66] loss=0.810442 avg=0.818756 VRAM=38.8GiB | 56.3% done | ETA(epoch): 884s + [Epoch 281][30/66] loss=0.931466 avg=0.821848 VRAM=39.0GiB | 56.3% done | ETA(epoch): 692s + [Epoch 281][40/66] loss=0.751986 avg=0.819949 VRAM=38.8GiB | 56.3% done | ETA(epoch): 500s + [Epoch 281][50/66] loss=0.778917 avg=0.831414 VRAM=39.0GiB | 56.4% done | ETA(epoch): 308s + [Epoch 281][60/66] loss=0.741815 avg=0.836938 VRAM=38.8GiB | 56.4% done | ETA(epoch): 115s + Train loss: 0.840117 (1268.5s) ETA: 5029min + Val loss: 0.802216 [t_0.0-0.2=1.0847 t_0.2-0.4=1.0163 t_0.4-0.6=0.8330 t_0.6-0.8=0.7104 t_0.8-1.0=0.6214] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0281 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 282/499 (56% done) --- + [Epoch 282][10/66] loss=0.919471 avg=0.818219 VRAM=39.0GiB | 56.4% done | ETA(epoch): 1076s + [Epoch 282][20/66] loss=0.844150 avg=0.847388 VRAM=38.8GiB | 56.5% done | ETA(epoch): 884s + [Epoch 282][30/66] loss=1.012037 avg=0.851276 VRAM=39.0GiB | 56.5% done | ETA(epoch): 692s + [Epoch 282][40/66] loss=0.917729 avg=0.848238 VRAM=38.8GiB | 56.5% done | ETA(epoch): 500s + [Epoch 282][50/66] loss=0.849491 avg=0.851965 VRAM=39.0GiB | 56.6% done | ETA(epoch): 308s + [Epoch 282][60/66] loss=1.007059 avg=0.854115 VRAM=38.8GiB | 56.6% done | ETA(epoch): 115s + Train loss: 0.852757 (1268.5s) ETA: 5006min + Val loss: 0.843588 [t_0.0-0.2=1.0956 t_0.2-0.4=0.9826 t_0.4-0.6=0.8078 t_0.6-0.8=0.6876 t_0.8-1.0=0.6291] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 283/499 (57% done) --- + [Epoch 283][10/66] loss=0.914820 avg=0.837658 VRAM=39.0GiB | 56.6% done | ETA(epoch): 1075s + [Epoch 283][20/66] loss=0.973629 avg=0.867762 VRAM=38.8GiB | 56.7% done | ETA(epoch): 884s + [Epoch 283][30/66] loss=0.926779 avg=0.879033 VRAM=39.0GiB | 56.7% done | ETA(epoch): 692s + [Epoch 283][40/66] loss=0.823342 avg=0.878603 VRAM=38.8GiB | 56.7% done | ETA(epoch): 500s + [Epoch 283][50/66] loss=0.832189 avg=0.874116 VRAM=39.0GiB | 56.8% done | ETA(epoch): 308s + [Epoch 283][60/66] loss=0.815526 avg=0.868852 VRAM=38.8GiB | 56.8% done | ETA(epoch): 115s + Train loss: 0.869609 (1270.1s) ETA: 4982min + Val loss: 0.878463 [t_0.0-0.2=1.0535 t_0.2-0.4=1.0069 t_0.4-0.6=0.8437 t_0.6-0.8=0.7012 t_0.8-1.0=0.6159] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 284/499 (57% done) --- + [Epoch 284][10/66] loss=0.838119 avg=0.871620 VRAM=39.0GiB | 56.8% done | ETA(epoch): 1079s + [Epoch 284][20/66] loss=0.770596 avg=0.852998 VRAM=38.8GiB | 56.9% done | ETA(epoch): 886s + [Epoch 284][30/66] loss=0.868687 avg=0.847714 VRAM=39.0GiB | 56.9% done | ETA(epoch): 694s + [Epoch 284][40/66] loss=0.803433 avg=0.835363 VRAM=38.8GiB | 56.9% done | ETA(epoch): 501s + [Epoch 284][50/66] loss=0.826626 avg=0.835239 VRAM=39.0GiB | 57.0% done | ETA(epoch): 308s + [Epoch 284][60/66] loss=0.995451 avg=0.845706 VRAM=38.8GiB | 57.0% done | ETA(epoch): 116s + Train loss: 0.845978 (1272.2s) ETA: 4959min + Val loss: 0.851347 [t_0.0-0.2=1.0756 t_0.2-0.4=0.9826 t_0.4-0.6=0.8272 t_0.6-0.8=0.7098 t_0.8-1.0=0.6072] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0284 + Deleted old checkpoint: checkpoint_epoch_0281 +[MEM @ epoch 284 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 285/499 (57% done) --- + [Epoch 285][10/66] loss=0.999366 avg=0.865918 VRAM=39.0GiB | 57.0% done | ETA(epoch): 1078s + [Epoch 285][20/66] loss=0.883895 avg=0.846955 VRAM=38.8GiB | 57.1% done | ETA(epoch): 886s + [Epoch 285][30/66] loss=0.855750 avg=0.856299 VRAM=39.0GiB | 57.1% done | ETA(epoch): 694s + [Epoch 285][40/66] loss=0.780680 avg=0.839858 VRAM=38.8GiB | 57.1% done | ETA(epoch): 502s + [Epoch 285][50/66] loss=0.865396 avg=0.838451 VRAM=39.0GiB | 57.2% done | ETA(epoch): 310s + [Epoch 285][60/66] loss=0.915509 avg=0.836366 VRAM=38.8GiB | 57.2% done | ETA(epoch): 116s + Train loss: 0.835171 (1278.1s) ETA: 4936min + Val loss: 0.820478 [t_0.0-0.2=1.0716 t_0.2-0.4=1.0447 t_0.4-0.6=0.8377 t_0.6-0.8=0.7212 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 286/499 (57% done) --- + [Epoch 286][10/66] loss=0.853624 avg=0.870733 VRAM=39.0GiB | 57.2% done | ETA(epoch): 1079s + [Epoch 286][20/66] loss=0.673933 avg=0.845440 VRAM=38.8GiB | 57.3% done | ETA(epoch): 887s + [Epoch 286][30/66] loss=0.843508 avg=0.839924 VRAM=39.0GiB | 57.3% done | ETA(epoch): 694s + [Epoch 286][40/66] loss=0.808860 avg=0.845679 VRAM=38.8GiB | 57.3% done | ETA(epoch): 503s + [Epoch 286][50/66] loss=0.648824 avg=0.833386 VRAM=39.0GiB | 57.4% done | ETA(epoch): 309s + [Epoch 286][60/66] loss=0.894029 avg=0.837965 VRAM=38.8GiB | 57.4% done | ETA(epoch): 116s + Train loss: 0.832988 (1275.2s) ETA: 4913min + Val loss: 0.834876 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9779 t_0.4-0.6=0.8490 t_0.6-0.8=0.7109 t_0.8-1.0=0.6183] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 287/499 (57% done) --- + [Epoch 287][10/66] loss=1.006057 avg=0.922503 VRAM=39.0GiB | 57.4% done | ETA(epoch): 1081s + [Epoch 287][20/66] loss=0.702346 avg=0.880769 VRAM=38.8GiB | 57.5% done | ETA(epoch): 888s + [Epoch 287][30/66] loss=0.905512 avg=0.876363 VRAM=39.0GiB | 57.5% done | ETA(epoch): 695s + [Epoch 287][40/66] loss=0.743614 avg=0.860513 VRAM=38.8GiB | 57.5% done | ETA(epoch): 502s + [Epoch 287][50/66] loss=0.815882 avg=0.850629 VRAM=39.0GiB | 57.6% done | ETA(epoch): 309s + [Epoch 287][60/66] loss=0.820020 avg=0.855046 VRAM=38.8GiB | 57.6% done | ETA(epoch): 116s + Train loss: 0.855409 (1274.3s) ETA: 4889min + Val loss: 0.854407 [t_0.0-0.2=1.0709 t_0.2-0.4=0.9943 t_0.4-0.6=0.8354 t_0.6-0.8=0.7048 t_0.8-1.0=0.6303] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 288/499 (58% done) --- + [Epoch 288][10/66] loss=0.775863 avg=0.826718 VRAM=39.0GiB | 57.6% done | ETA(epoch): 1080s + [Epoch 288][20/66] loss=0.876997 avg=0.847353 VRAM=38.8GiB | 57.7% done | ETA(epoch): 887s + [Epoch 288][30/66] loss=0.745504 avg=0.843466 VRAM=39.0GiB | 57.7% done | ETA(epoch): 695s + [Epoch 288][40/66] loss=0.856577 avg=0.840113 VRAM=38.8GiB | 57.7% done | ETA(epoch): 502s + [Epoch 288][50/66] loss=0.956075 avg=0.844844 VRAM=39.0GiB | 57.8% done | ETA(epoch): 309s + [Epoch 288][60/66] loss=0.913514 avg=0.843582 VRAM=38.8GiB | 57.8% done | ETA(epoch): 116s + Train loss: 0.840008 (1274.3s) ETA: 4866min + Val loss: 0.875407 [t_0.0-0.2=1.0495 t_0.2-0.4=0.9701 t_0.4-0.6=0.8539 t_0.6-0.8=0.7335 t_0.8-1.0=0.6235] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 289/499 (58% done) --- + [Epoch 289][10/66] loss=0.745452 avg=0.846433 VRAM=39.0GiB | 57.8% done | ETA(epoch): 1082s + [Epoch 289][20/66] loss=0.752103 avg=0.840789 VRAM=38.8GiB | 57.9% done | ETA(epoch): 893s + [Epoch 289][30/66] loss=0.852585 avg=0.838917 VRAM=39.0GiB | 57.9% done | ETA(epoch): 698s + [Epoch 289][40/66] loss=1.002992 avg=0.844434 VRAM=38.8GiB | 57.9% done | ETA(epoch): 503s + [Epoch 289][50/66] loss=0.874276 avg=0.845112 VRAM=39.0GiB | 58.0% done | ETA(epoch): 310s + [Epoch 289][60/66] loss=0.879415 avg=0.846038 VRAM=38.8GiB | 58.0% done | ETA(epoch): 116s + Train loss: 0.848131 (1276.6s) ETA: 4843min + Val loss: 0.896205 [t_0.0-0.2=1.0569 t_0.2-0.4=1.0047 t_0.4-0.6=0.8621 t_0.6-0.8=0.6913 t_0.8-1.0=0.6402] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 290/499 (58% done) --- + [Epoch 290][10/66] loss=0.969454 avg=0.857578 VRAM=39.0GiB | 58.0% done | ETA(epoch): 1079s + [Epoch 290][20/66] loss=0.838725 avg=0.848788 VRAM=38.8GiB | 58.1% done | ETA(epoch): 888s + [Epoch 290][30/66] loss=0.888046 avg=0.839343 VRAM=39.0GiB | 58.1% done | ETA(epoch): 695s + [Epoch 290][40/66] loss=0.785248 avg=0.846368 VRAM=38.8GiB | 58.1% done | ETA(epoch): 502s + [Epoch 290][50/66] loss=0.935677 avg=0.849615 VRAM=39.0GiB | 58.2% done | ETA(epoch): 309s + [Epoch 290][60/66] loss=0.817102 avg=0.845248 VRAM=38.8GiB | 58.2% done | ETA(epoch): 116s + Train loss: 0.843470 (1273.5s) ETA: 4820min + Val loss: 0.859679 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0162 t_0.4-0.6=0.8377 t_0.6-0.8=0.7143 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 291/499 (58% done) --- + [Epoch 291][10/66] loss=0.849001 avg=0.822272 VRAM=39.0GiB | 58.2% done | ETA(epoch): 1080s + [Epoch 291][20/66] loss=0.824470 avg=0.841302 VRAM=38.8GiB | 58.3% done | ETA(epoch): 887s + [Epoch 291][30/66] loss=0.910809 avg=0.855308 VRAM=39.0GiB | 58.3% done | ETA(epoch): 695s + [Epoch 291][40/66] loss=0.832597 avg=0.862625 VRAM=38.8GiB | 58.3% done | ETA(epoch): 502s + [Epoch 291][50/66] loss=0.934042 avg=0.854194 VRAM=39.0GiB | 58.4% done | ETA(epoch): 309s + [Epoch 291][60/66] loss=0.772026 avg=0.847639 VRAM=38.8GiB | 58.4% done | ETA(epoch): 118s + Train loss: 0.847275 (1291.6s) ETA: 4797min + Val loss: 0.842980 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9922 t_0.4-0.6=0.8552 t_0.6-0.8=0.6653 t_0.8-1.0=0.6163] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 292/499 (58% done) --- + [Epoch 292][10/66] loss=0.964820 avg=0.893991 VRAM=39.0GiB | 58.4% done | ETA(epoch): 1082s + [Epoch 292][20/66] loss=0.874234 avg=0.894364 VRAM=38.8GiB | 58.5% done | ETA(epoch): 888s + [Epoch 292][30/66] loss=0.694967 avg=0.871321 VRAM=39.0GiB | 58.5% done | ETA(epoch): 695s + [Epoch 292][40/66] loss=0.799809 avg=0.862348 VRAM=38.8GiB | 58.5% done | ETA(epoch): 502s + [Epoch 292][50/66] loss=0.908594 avg=0.866892 VRAM=39.0GiB | 58.6% done | ETA(epoch): 309s + [Epoch 292][60/66] loss=0.909415 avg=0.861647 VRAM=38.8GiB | 58.6% done | ETA(epoch): 116s + Train loss: 0.860202 (1274.3s) ETA: 4773min + Val loss: 0.861297 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9883 t_0.4-0.6=0.8308 t_0.6-0.8=0.6998 t_0.8-1.0=0.6121] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 293/499 (59% done) --- + [Epoch 293][10/66] loss=0.821360 avg=0.861246 VRAM=39.0GiB | 58.6% done | ETA(epoch): 1082s + [Epoch 293][20/66] loss=0.920642 avg=0.854450 VRAM=38.8GiB | 58.7% done | ETA(epoch): 888s + [Epoch 293][30/66] loss=0.892188 avg=0.860480 VRAM=39.0GiB | 58.7% done | ETA(epoch): 695s + [Epoch 293][40/66] loss=0.842309 avg=0.857628 VRAM=38.8GiB | 58.7% done | ETA(epoch): 502s + [Epoch 293][50/66] loss=0.714278 avg=0.845893 VRAM=39.0GiB | 58.8% done | ETA(epoch): 309s + [Epoch 293][60/66] loss=0.782880 avg=0.844676 VRAM=38.8GiB | 58.8% done | ETA(epoch): 116s + Train loss: 0.847004 (1275.0s) ETA: 4750min + Val loss: 0.847499 [t_0.0-0.2=1.0802 t_0.2-0.4=0.9959 t_0.4-0.6=0.8420 t_0.6-0.8=0.6823 t_0.8-1.0=0.6135] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 294/499 (59% done) --- + [Epoch 294][10/66] loss=0.716704 avg=0.872982 VRAM=39.0GiB | 58.8% done | ETA(epoch): 1084s + [Epoch 294][20/66] loss=0.908018 avg=0.855069 VRAM=38.8GiB | 58.9% done | ETA(epoch): 891s + [Epoch 294][30/66] loss=0.885165 avg=0.856289 VRAM=39.0GiB | 58.9% done | ETA(epoch): 696s + [Epoch 294][40/66] loss=0.879383 avg=0.846029 VRAM=38.8GiB | 58.9% done | ETA(epoch): 506s + [Epoch 294][50/66] loss=0.835947 avg=0.840339 VRAM=39.0GiB | 59.0% done | ETA(epoch): 311s + [Epoch 294][60/66] loss=0.771846 avg=0.842362 VRAM=38.8GiB | 59.0% done | ETA(epoch): 116s + Train loss: 0.845778 (1279.3s) ETA: 4727min + Val loss: 0.849342 [t_0.0-0.2=1.0648 t_0.2-0.4=0.9767 t_0.4-0.6=0.8436 t_0.6-0.8=0.6880 t_0.8-1.0=0.6359] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 295/499 (59% done) --- + [Epoch 295][10/66] loss=0.870425 avg=0.858258 VRAM=39.0GiB | 59.0% done | ETA(epoch): 1079s + [Epoch 295][20/66] loss=0.975546 avg=0.860755 VRAM=38.8GiB | 59.1% done | ETA(epoch): 887s + [Epoch 295][30/66] loss=0.955045 avg=0.862530 VRAM=39.0GiB | 59.1% done | ETA(epoch): 694s + [Epoch 295][40/66] loss=0.946283 avg=0.856903 VRAM=38.8GiB | 59.1% done | ETA(epoch): 501s + [Epoch 295][50/66] loss=0.904608 avg=0.853144 VRAM=39.0GiB | 59.2% done | ETA(epoch): 309s + [Epoch 295][60/66] loss=0.793803 avg=0.853284 VRAM=38.8GiB | 59.2% done | ETA(epoch): 116s + Train loss: 0.853579 (1273.1s) ETA: 4704min + Val loss: 0.821507 [t_0.0-0.2=1.0631 t_0.2-0.4=0.9622 t_0.4-0.6=0.8617 t_0.6-0.8=0.6964 t_0.8-1.0=0.6305] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 296/499 (59% done) --- + [Epoch 296][10/66] loss=0.891018 avg=0.865981 VRAM=39.0GiB | 59.2% done | ETA(epoch): 1079s + [Epoch 296][20/66] loss=0.887534 avg=0.858667 VRAM=38.8GiB | 59.3% done | ETA(epoch): 886s + [Epoch 296][30/66] loss=0.905805 avg=0.856503 VRAM=39.0GiB | 59.3% done | ETA(epoch): 694s + [Epoch 296][40/66] loss=1.000472 avg=0.860054 VRAM=38.8GiB | 59.3% done | ETA(epoch): 501s + [Epoch 296][50/66] loss=1.016027 avg=0.863864 VRAM=39.0GiB | 59.4% done | ETA(epoch): 308s + [Epoch 296][60/66] loss=0.776970 avg=0.864235 VRAM=38.8GiB | 59.4% done | ETA(epoch): 116s + Train loss: 0.863465 (1273.1s) ETA: 4681min + Val loss: 0.855673 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9489 t_0.4-0.6=0.8217 t_0.6-0.8=0.7188 t_0.8-1.0=0.6292] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 297/499 (59% done) --- + [Epoch 297][10/66] loss=0.927723 avg=0.835005 VRAM=39.0GiB | 59.4% done | ETA(epoch): 1082s + [Epoch 297][20/66] loss=0.752176 avg=0.848138 VRAM=38.8GiB | 59.5% done | ETA(epoch): 889s + [Epoch 297][30/66] loss=0.867263 avg=0.844730 VRAM=39.0GiB | 59.5% done | ETA(epoch): 695s + [Epoch 297][40/66] loss=0.860344 avg=0.847791 VRAM=38.8GiB | 59.5% done | ETA(epoch): 502s + [Epoch 297][50/66] loss=0.856070 avg=0.843595 VRAM=39.0GiB | 59.6% done | ETA(epoch): 309s + [Epoch 297][60/66] loss=0.777858 avg=0.843801 VRAM=38.8GiB | 59.6% done | ETA(epoch): 116s + Train loss: 0.844797 (1273.1s) ETA: 4657min + Val loss: 0.855523 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0032 t_0.4-0.6=0.8187 t_0.6-0.8=0.7029 t_0.8-1.0=0.6074] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 298/499 (60% done) --- + [Epoch 298][10/66] loss=0.865928 avg=0.870704 VRAM=39.0GiB | 59.6% done | ETA(epoch): 1078s + [Epoch 298][20/66] loss=0.913483 avg=0.846398 VRAM=38.8GiB | 59.7% done | ETA(epoch): 886s + [Epoch 298][30/66] loss=0.915743 avg=0.840241 VRAM=39.0GiB | 59.7% done | ETA(epoch): 693s + [Epoch 298][40/66] loss=0.913151 avg=0.849114 VRAM=38.8GiB | 59.7% done | ETA(epoch): 501s + [Epoch 298][50/66] loss=0.807032 avg=0.848064 VRAM=39.0GiB | 59.8% done | ETA(epoch): 308s + [Epoch 298][60/66] loss=0.927697 avg=0.850329 VRAM=38.8GiB | 59.8% done | ETA(epoch): 116s + Train loss: 0.852972 (1271.2s) ETA: 4634min + Val loss: 0.846393 [t_0.0-0.2=1.0572 t_0.2-0.4=0.9750 t_0.4-0.6=0.8787 t_0.6-0.8=0.7288 t_0.8-1.0=0.6212] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 299/499 (60% done) --- + [Epoch 299][10/66] loss=0.893925 avg=0.885499 VRAM=39.0GiB | 59.8% done | ETA(epoch): 1079s + [Epoch 299][20/66] loss=0.747378 avg=0.875647 VRAM=38.8GiB | 59.9% done | ETA(epoch): 887s + [Epoch 299][30/66] loss=0.722678 avg=0.868421 VRAM=39.0GiB | 59.9% done | ETA(epoch): 694s + [Epoch 299][40/66] loss=0.886680 avg=0.849058 VRAM=38.8GiB | 59.9% done | ETA(epoch): 501s + [Epoch 299][50/66] loss=0.891339 avg=0.848694 VRAM=39.0GiB | 60.0% done | ETA(epoch): 308s + [Epoch 299][60/66] loss=0.738250 avg=0.841181 VRAM=38.8GiB | 60.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 19799 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 19799 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 19799 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 19799 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 19799 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 19799 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 19799 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 19799 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 19799 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 19799 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.843606 (2306.8s) ETA: 4622min + Val loss: 0.835320 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9564 t_0.4-0.6=0.8641 t_0.6-0.8=0.6977 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 300/499 (60% done) --- diff --git a/lora_512_dim_512_H100_a/train.log b/lora_512_dim_512_H100_a/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e933f8307051b3d26878edd62a69a59f884b3688 --- /dev/null +++ b/lora_512_dim_512_H100_a/train.log @@ -0,0 +1,4398 @@ + +====================================================================== +Command: train_script_ablation_a.py --name lora_512_dim_512_H100_a --batch-size 6 --adapter-lr 1e-3 --num-epochs 300 --lora-lr 1e-4 --adapter-mlp --lora-rank 512 --fusion-dim 512 --seed 42 +====================================================================== +Run dir : output/lora_512_dim_512_H100_a +Log file: output/lora_512_dim_512_H100_a/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run ix8fn97c (0.3s) + m wandb: setting up run ix8fn97c (0.3s) + m wandb: setting up run ix8fn97c (0.3s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260513_100116-ix8fn97c +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_512_dim_512_H100_a-train-20260513-100116 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_512_dim_512_H100_a +wandb: View run at https://wandb.ai/hku-xg-boost/lora_512_dim_512_H100_a/runs/ix8fn97c + +Run Summary: + name lora_512_dim_512_H100_a + fusion MLP (lightweight) + LoRA ON (rank=512) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_512_dim_512_H100_a + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 512 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 300 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_512_dim_512_H100_a + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 13.8/188.5 GiB (7.3%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.6s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 14.2/188.5 GiB (7.5%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (65.7s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 16.0/188.5 GiB (8.5%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating Ablation Adapter (a: seg+depth concat, no fusion) +============================================================ + Ablation mode: (a) Native concat baseline (seg + depth, no HDC^2A fusion) +HDC²A: 3.1M params +Control: 0.0M params +Total trainable: 3.1M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=512, alpha=512.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 157.29M + +Parameter Statistics: + HDC²A Adapter: total=3.1M trainable=3.1M + ControlNet (frozen): total=4290.8M LoRA trainable=157.29M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 3.1M + LoRA 157.29M = 160.35M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 15 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~4800 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0364, std=0.5391 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.367754 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 45.57 GiB + [test 4/4] Gradient flow check... + HDC²A: 15/15 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + conv_stem.6.weight: 0.047852 + conv_stem.3.weight: 0.027588 + proj.weight: 0.026367 + conv_stem.0.weight: 0.004974 + proj.bias: 0.002472 + Test result: PASSED +[MEM @ after test] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.2/93.1 GiB (41.1%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 210.437 GFLOPs + transformer: 274103.488 GFLOPs + total: 274313.924 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 300 epochs × 66 steps = 19800 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 1980, 3960, 5940, 7920, 9900, 11879, 13859, 15839, 17819, 19799] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/299 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/66] loss=0.880929 avg=0.897211 VRAM=39.5GiB | 0.1% done | ETA(epoch): 6735s + [Epoch 0][20/66] loss=1.001402 avg=0.924587 VRAM=39.2GiB | 0.1% done | ETA(epoch): 3212s + [Epoch 0][30/66] loss=0.922406 avg=0.914367 VRAM=39.5GiB | 0.2% done | ETA(epoch): 1909s + [Epoch 0][40/66] loss=0.936713 avg=0.924003 VRAM=39.2GiB | 0.2% done | ETA(epoch): 1160s + [Epoch 0][50/66] loss=0.886340 avg=0.923880 VRAM=39.5GiB | 0.3% done | ETA(epoch): 633s + [Epoch 0][60/66] loss=0.921814 avg=0.925786 VRAM=39.2GiB | 0.3% done | ETA(epoch): 217s + Train loss: 0.925080 (2288.5s) ETA: 11428min + Val loss: 0.963952 [t_0.0-0.2=1.0859 t_0.2-0.4=1.0174 t_0.4-0.6=0.8960 t_0.6-0.8=0.7738 t_0.8-1.0=0.8800] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 1/299 (0% done) --- + [Epoch 1][10/66] loss=0.984193 avg=0.893325 VRAM=39.5GiB | 0.4% done | ETA(epoch): 1084s + [Epoch 1][20/66] loss=0.980459 avg=0.899708 VRAM=39.2GiB | 0.4% done | ETA(epoch): 891s + [Epoch 1][30/66] loss=0.994144 avg=0.905975 VRAM=39.5GiB | 0.5% done | ETA(epoch): 699s + [Epoch 1][40/66] loss=0.969196 avg=0.907178 VRAM=39.2GiB | 0.5% done | ETA(epoch): 505s + [Epoch 1][50/66] loss=0.904378 avg=0.901161 VRAM=39.5GiB | 0.6% done | ETA(epoch): 311s + [Epoch 1][60/66] loss=0.865294 avg=0.903726 VRAM=39.2GiB | 0.6% done | ETA(epoch): 116s + Train loss: 0.903384 (1280.0s) ETA: 9117min + Val loss: 0.920658 [t_0.0-0.2=1.0826 t_0.2-0.4=1.0257 t_0.4-0.6=0.8835 t_0.6-0.8=0.7501 t_0.8-1.0=0.9449] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 15.9/188.5 GiB (8.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 2/299 (1% done) --- + [Epoch 2][10/66] loss=0.947156 avg=0.900156 VRAM=39.5GiB | 0.7% done | ETA(epoch): 1080s + [Epoch 2][20/66] loss=0.969899 avg=0.906328 VRAM=39.2GiB | 0.8% done | ETA(epoch): 885s + [Epoch 2][30/66] loss=0.992588 avg=0.910230 VRAM=39.5GiB | 0.8% done | ETA(epoch): 694s + [Epoch 2][40/66] loss=0.965855 avg=0.915093 VRAM=39.2GiB | 0.9% done | ETA(epoch): 502s + [Epoch 2][50/66] loss=0.868376 avg=0.911796 VRAM=39.5GiB | 0.9% done | ETA(epoch): 309s + [Epoch 2][60/66] loss=1.085127 avg=0.910869 VRAM=39.2GiB | 1.0% done | ETA(epoch): 116s + Train loss: 0.908334 (1278.0s) ETA: 8323min + Val loss: 0.950349 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9957 t_0.4-0.6=0.8789 t_0.6-0.8=0.7670 t_0.8-1.0=0.9739] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 3/299 (1% done) --- + [Epoch 3][10/66] loss=0.822196 avg=0.907717 VRAM=39.5GiB | 1.1% done | ETA(epoch): 1086s + [Epoch 3][20/66] loss=0.844118 avg=0.899104 VRAM=39.2GiB | 1.1% done | ETA(epoch): 893s + [Epoch 3][30/66] loss=0.833845 avg=0.888553 VRAM=39.5GiB | 1.2% done | ETA(epoch): 699s + [Epoch 3][40/66] loss=0.861960 avg=0.888473 VRAM=39.2GiB | 1.2% done | ETA(epoch): 505s + [Epoch 3][50/66] loss=0.960295 avg=0.887634 VRAM=39.5GiB | 1.3% done | ETA(epoch): 311s + [Epoch 3][60/66] loss=0.971817 avg=0.890308 VRAM=39.2GiB | 1.3% done | ETA(epoch): 117s + Train loss: 0.891099 (1284.1s) ETA: 7922min + Val loss: 0.900598 [t_0.0-0.2=1.0778 t_0.2-0.4=1.0329 t_0.4-0.6=0.8750 t_0.6-0.8=0.7494 t_0.8-1.0=0.8735] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 16.0/188.5 GiB (8.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 4/299 (1% done) --- + [Epoch 4][10/66] loss=0.885655 avg=0.858515 VRAM=39.5GiB | 1.4% done | ETA(epoch): 1087s + [Epoch 4][20/66] loss=0.843153 avg=0.864274 VRAM=39.2GiB | 1.4% done | ETA(epoch): 893s + [Epoch 4][30/66] loss=0.890647 avg=0.874644 VRAM=39.5GiB | 1.5% done | ETA(epoch): 699s + [Epoch 4][40/66] loss=0.831496 avg=0.873968 VRAM=39.2GiB | 1.5% done | ETA(epoch): 505s + [Epoch 4][50/66] loss=0.873911 avg=0.880034 VRAM=39.5GiB | 1.6% done | ETA(epoch): 311s + [Epoch 4][60/66] loss=0.833412 avg=0.884009 VRAM=39.2GiB | 1.6% done | ETA(epoch): 117s + Train loss: 0.881987 (1281.6s) ETA: 7672min + Val loss: 0.936316 [t_0.0-0.2=1.0844 t_0.2-0.4=1.0302 t_0.4-0.6=0.8370 t_0.6-0.8=0.7567 t_0.8-1.0=0.8314] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 15.9/188.5 GiB (8.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 5/299 (2% done) --- + [Epoch 5][10/66] loss=0.871049 avg=0.889201 VRAM=39.5GiB | 1.7% done | ETA(epoch): 1080s + [Epoch 5][20/66] loss=0.922662 avg=0.893384 VRAM=39.2GiB | 1.8% done | ETA(epoch): 889s + [Epoch 5][30/66] loss=0.926128 avg=0.892141 VRAM=39.5GiB | 1.8% done | ETA(epoch): 696s + [Epoch 5][40/66] loss=0.846111 avg=0.891650 VRAM=39.2GiB | 1.9% done | ETA(epoch): 503s + [Epoch 5][50/66] loss=0.852932 avg=0.888305 VRAM=39.5GiB | 1.9% done | ETA(epoch): 310s + [Epoch 5][60/66] loss=0.808257 avg=0.889100 VRAM=39.2GiB | 2.0% done | ETA(epoch): 116s + Train loss: 0.891208 (1278.9s) ETA: 7495min + Val loss: 0.913230 [t_0.0-0.2=1.0680 t_0.2-0.4=0.9431 t_0.4-0.6=0.8780 t_0.6-0.8=0.7562 t_0.8-1.0=0.8834] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 16.0/188.5 GiB (8.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 6/299 (2% done) --- + [Epoch 6][10/66] loss=0.782269 avg=0.867635 VRAM=39.5GiB | 2.1% done | ETA(epoch): 1085s + [Epoch 6][20/66] loss=0.897733 avg=0.862267 VRAM=39.2GiB | 2.1% done | ETA(epoch): 892s + [Epoch 6][30/66] loss=0.833670 avg=0.877700 VRAM=39.5GiB | 2.2% done | ETA(epoch): 698s + [Epoch 6][40/66] loss=0.914180 avg=0.885569 VRAM=39.2GiB | 2.2% done | ETA(epoch): 504s + [Epoch 6][50/66] loss=0.813455 avg=0.888158 VRAM=39.5GiB | 2.3% done | ETA(epoch): 310s + [Epoch 6][60/66] loss=0.899814 avg=0.885486 VRAM=39.2GiB | 2.3% done | ETA(epoch): 116s + Train loss: 0.885546 (1280.0s) ETA: 7362min + Val loss: 0.937772 [t_0.0-0.2=1.0849 t_0.2-0.4=1.0139 t_0.4-0.6=0.8623 t_0.6-0.8=0.7450 t_0.8-1.0=0.8897] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0006 +[MEM @ epoch 6 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 7/299 (2% done) --- + [Epoch 7][10/66] loss=0.915137 avg=0.907741 VRAM=39.5GiB | 2.4% done | ETA(epoch): 1087s + [Epoch 7][20/66] loss=0.915945 avg=0.905633 VRAM=39.2GiB | 2.4% done | ETA(epoch): 893s + [Epoch 7][30/66] loss=0.902303 avg=0.905896 VRAM=39.5GiB | 2.5% done | ETA(epoch): 699s + [Epoch 7][40/66] loss=1.021366 avg=0.900860 VRAM=39.2GiB | 2.5% done | ETA(epoch): 505s + [Epoch 7][50/66] loss=0.885502 avg=0.899246 VRAM=39.5GiB | 2.6% done | ETA(epoch): 311s + [Epoch 7][60/66] loss=0.995065 avg=0.895368 VRAM=39.2GiB | 2.6% done | ETA(epoch): 117s + Train loss: 0.895274 (1283.2s) ETA: 7258min + Val loss: 0.912153 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0092 t_0.4-0.6=0.9258 t_0.6-0.8=0.7429 t_0.8-1.0=0.7854] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 15.9/188.5 GiB (8.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 8/299 (3% done) --- + [Epoch 8][10/66] loss=0.861954 avg=0.904688 VRAM=39.5GiB | 2.7% done | ETA(epoch): 1083s + [Epoch 8][20/66] loss=1.017420 avg=0.893682 VRAM=39.2GiB | 2.8% done | ETA(epoch): 891s + [Epoch 8][30/66] loss=0.877673 avg=0.892669 VRAM=39.5GiB | 2.8% done | ETA(epoch): 697s + [Epoch 8][40/66] loss=0.833235 avg=0.896910 VRAM=39.2GiB | 2.9% done | ETA(epoch): 504s + [Epoch 8][50/66] loss=0.878058 avg=0.896461 VRAM=39.5GiB | 2.9% done | ETA(epoch): 310s + [Epoch 8][60/66] loss=0.907755 avg=0.892939 VRAM=39.2GiB | 3.0% done | ETA(epoch): 116s + Train loss: 0.896394 (1279.0s) ETA: 7171min + Val loss: 0.910639 [t_0.0-0.2=1.1037 t_0.2-0.4=1.0331 t_0.4-0.6=0.8552 t_0.6-0.8=0.7269 t_0.8-1.0=0.8040] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 9/299 (3% done) --- + [Epoch 9][10/66] loss=0.889748 avg=0.867319 VRAM=39.5GiB | 3.1% done | ETA(epoch): 1081s + [Epoch 9][20/66] loss=0.867459 avg=0.872508 VRAM=39.2GiB | 3.1% done | ETA(epoch): 890s + [Epoch 9][30/66] loss=0.884048 avg=0.888527 VRAM=39.5GiB | 3.2% done | ETA(epoch): 697s + [Epoch 9][40/66] loss=0.883553 avg=0.888166 VRAM=39.2GiB | 3.2% done | ETA(epoch): 504s + [Epoch 9][50/66] loss=0.819154 avg=0.883801 VRAM=39.5GiB | 3.3% done | ETA(epoch): 310s + [Epoch 9][60/66] loss=0.893636 avg=0.887183 VRAM=39.2GiB | 3.3% done | ETA(epoch): 117s + Train loss: 0.889496 (1282.0s) ETA: 7098min + Val loss: 0.912581 [t_0.0-0.2=1.0722 t_0.2-0.4=1.0036 t_0.4-0.6=0.8721 t_0.6-0.8=0.7167 t_0.8-1.0=0.8490] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 10/299 (3% done) --- + [Epoch 10][10/66] loss=0.876300 avg=0.875913 VRAM=39.5GiB | 3.4% done | ETA(epoch): 1082s + [Epoch 10][20/66] loss=0.883048 avg=0.888214 VRAM=39.2GiB | 3.4% done | ETA(epoch): 888s + [Epoch 10][30/66] loss=0.873250 avg=0.901162 VRAM=39.5GiB | 3.5% done | ETA(epoch): 695s + [Epoch 10][40/66] loss=0.869530 avg=0.893651 VRAM=39.2GiB | 3.5% done | ETA(epoch): 502s + [Epoch 10][50/66] loss=0.749424 avg=0.886792 VRAM=39.5GiB | 3.6% done | ETA(epoch): 309s + [Epoch 10][60/66] loss=0.794085 avg=0.889215 VRAM=39.2GiB | 3.6% done | ETA(epoch): 116s + Train loss: 0.889945 (1275.6s) ETA: 7031min + Val loss: 0.913138 [t_0.0-0.2=1.0730 t_0.2-0.4=1.0012 t_0.4-0.6=0.8557 t_0.6-0.8=0.7289 t_0.8-1.0=0.8233] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0010 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 11/299 (4% done) --- + [Epoch 11][10/66] loss=0.788777 avg=0.868580 VRAM=39.5GiB | 3.7% done | ETA(epoch): 1086s + [Epoch 11][20/66] loss=0.918070 avg=0.891092 VRAM=39.2GiB | 3.8% done | ETA(epoch): 893s + [Epoch 11][30/66] loss=0.903479 avg=0.893922 VRAM=39.5GiB | 3.8% done | ETA(epoch): 699s + [Epoch 11][40/66] loss=0.870053 avg=0.891905 VRAM=39.2GiB | 3.9% done | ETA(epoch): 505s + [Epoch 11][50/66] loss=0.892059 avg=0.887977 VRAM=39.5GiB | 3.9% done | ETA(epoch): 311s + [Epoch 11][60/66] loss=0.833060 avg=0.882113 VRAM=39.2GiB | 4.0% done | ETA(epoch): 117s + Train loss: 0.881131 (1282.4s) ETA: 6974min + Val loss: 0.888319 [t_0.0-0.2=1.0671 t_0.2-0.4=1.0119 t_0.4-0.6=0.8613 t_0.6-0.8=0.7100 t_0.8-1.0=0.7910] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0011 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 16.0/188.5 GiB (8.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 12/299 (4% done) --- + [Epoch 12][10/66] loss=0.818324 avg=0.888838 VRAM=39.5GiB | 4.1% done | ETA(epoch): 1087s + [Epoch 12][20/66] loss=0.806999 avg=0.885679 VRAM=39.2GiB | 4.1% done | ETA(epoch): 894s + [Epoch 12][30/66] loss=1.020445 avg=0.898417 VRAM=39.5GiB | 4.2% done | ETA(epoch): 700s + [Epoch 12][40/66] loss=0.794544 avg=0.892114 VRAM=39.2GiB | 4.2% done | ETA(epoch): 506s + [Epoch 12][50/66] loss=0.837014 avg=0.896104 VRAM=39.5GiB | 4.3% done | ETA(epoch): 311s + [Epoch 12][60/66] loss=0.897105 avg=0.896911 VRAM=39.2GiB | 4.3% done | ETA(epoch): 117s + Train loss: 0.895882 (1284.4s) ETA: 6923min + Val loss: 0.876683 [t_0.0-0.2=1.0621 t_0.2-0.4=0.9899 t_0.4-0.6=0.8616 t_0.6-0.8=0.7607 t_0.8-1.0=0.7366] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0012 (BEST) + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 13/299 (4% done) --- + [Epoch 13][10/66] loss=0.867455 avg=0.902430 VRAM=39.5GiB | 4.4% done | ETA(epoch): 1087s + [Epoch 13][20/66] loss=0.900021 avg=0.889886 VRAM=39.2GiB | 4.4% done | ETA(epoch): 894s + [Epoch 13][30/66] loss=0.921272 avg=0.893393 VRAM=39.5GiB | 4.5% done | ETA(epoch): 700s + [Epoch 13][40/66] loss=0.931508 avg=0.898939 VRAM=39.2GiB | 4.5% done | ETA(epoch): 506s + [Epoch 13][50/66] loss=0.796335 avg=0.896292 VRAM=39.5GiB | 4.6% done | ETA(epoch): 311s + [Epoch 13][60/66] loss=0.854240 avg=0.894451 VRAM=39.2GiB | 4.6% done | ETA(epoch): 117s + Train loss: 0.888608 (1283.2s) ETA: 6876min + Val loss: 0.901349 [t_0.0-0.2=1.0721 t_0.2-0.4=1.0144 t_0.4-0.6=0.8553 t_0.6-0.8=0.7503 t_0.8-1.0=0.7695] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 14/299 (5% done) --- + [Epoch 14][10/66] loss=0.781435 avg=0.882337 VRAM=39.5GiB | 4.7% done | ETA(epoch): 1088s + [Epoch 14][20/66] loss=0.871119 avg=0.869958 VRAM=39.2GiB | 4.8% done | ETA(epoch): 904s + [Epoch 14][30/66] loss=0.779965 avg=0.860087 VRAM=39.5GiB | 4.8% done | ETA(epoch): 706s + [Epoch 14][40/66] loss=0.863474 avg=0.867898 VRAM=39.2GiB | 4.9% done | ETA(epoch): 509s + [Epoch 14][50/66] loss=0.819141 avg=0.868834 VRAM=39.5GiB | 4.9% done | ETA(epoch): 312s + [Epoch 14][60/66] loss=0.887754 avg=0.868280 VRAM=39.2GiB | 5.0% done | ETA(epoch): 117s + Train loss: 0.871201 (1284.7s) ETA: 6832min + Val loss: 0.864358 [t_0.0-0.2=1.0699 t_0.2-0.4=0.9895 t_0.4-0.6=0.8728 t_0.6-0.8=0.7212 t_0.8-1.0=0.7184] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0014 (BEST) + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 15/299 (5% done) --- + [Epoch 15][10/66] loss=0.810657 avg=0.879341 VRAM=39.5GiB | 5.1% done | ETA(epoch): 1087s + [Epoch 15][20/66] loss=0.945712 avg=0.896192 VRAM=39.2GiB | 5.1% done | ETA(epoch): 892s + [Epoch 15][30/66] loss=0.833317 avg=0.894978 VRAM=39.5GiB | 5.2% done | ETA(epoch): 697s + [Epoch 15][40/66] loss=0.931318 avg=0.891311 VRAM=39.2GiB | 5.2% done | ETA(epoch): 504s + [Epoch 15][50/66] loss=0.879442 avg=0.897010 VRAM=39.5GiB | 5.3% done | ETA(epoch): 310s + [Epoch 15][60/66] loss=0.744307 avg=0.891735 VRAM=39.2GiB | 5.3% done | ETA(epoch): 116s + Train loss: 0.888878 (1280.9s) ETA: 6790min + Val loss: 0.886268 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9819 t_0.4-0.6=0.8816 t_0.6-0.8=0.7499 t_0.8-1.0=0.7139] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 16/299 (5% done) --- + [Epoch 16][10/66] loss=0.875266 avg=0.878433 VRAM=39.5GiB | 5.4% done | ETA(epoch): 1083s + [Epoch 16][20/66] loss=0.969930 avg=0.887590 VRAM=39.2GiB | 5.4% done | ETA(epoch): 890s + [Epoch 16][30/66] loss=0.910370 avg=0.873232 VRAM=39.5GiB | 5.5% done | ETA(epoch): 698s + [Epoch 16][40/66] loss=0.833804 avg=0.872283 VRAM=39.2GiB | 5.5% done | ETA(epoch): 504s + [Epoch 16][50/66] loss=1.011794 avg=0.881438 VRAM=39.5GiB | 5.6% done | ETA(epoch): 310s + [Epoch 16][60/66] loss=0.939938 avg=0.884710 VRAM=39.2GiB | 5.6% done | ETA(epoch): 117s + Train loss: 0.879859 (1285.1s) ETA: 6751min + Val loss: 0.866044 [t_0.0-0.2=1.0860 t_0.2-0.4=0.9959 t_0.4-0.6=0.8487 t_0.6-0.8=0.7462 t_0.8-1.0=0.7136] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 16.5/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 17/299 (6% done) --- + [Epoch 17][10/66] loss=0.854120 avg=0.841382 VRAM=39.5GiB | 5.7% done | ETA(epoch): 1083s + [Epoch 17][20/66] loss=0.836033 avg=0.864190 VRAM=39.2GiB | 5.8% done | ETA(epoch): 888s + [Epoch 17][30/66] loss=0.939276 avg=0.868050 VRAM=39.5GiB | 5.8% done | ETA(epoch): 694s + [Epoch 17][40/66] loss=0.865083 avg=0.865445 VRAM=39.2GiB | 5.9% done | ETA(epoch): 502s + [Epoch 17][50/66] loss=0.971851 avg=0.872927 VRAM=39.5GiB | 5.9% done | ETA(epoch): 310s + [Epoch 17][60/66] loss=0.763240 avg=0.872637 VRAM=39.2GiB | 6.0% done | ETA(epoch): 116s + Train loss: 0.868535 (1279.3s) ETA: 6712min + Val loss: 0.857086 [t_0.0-0.2=1.0810 t_0.2-0.4=0.9975 t_0.4-0.6=0.8878 t_0.6-0.8=0.7042 t_0.8-1.0=0.7234] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0017 (BEST) + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 18/299 (6% done) --- + [Epoch 18][10/66] loss=0.878696 avg=0.850861 VRAM=39.5GiB | 6.1% done | ETA(epoch): 1084s + [Epoch 18][20/66] loss=0.884363 avg=0.855901 VRAM=39.2GiB | 6.1% done | ETA(epoch): 888s + [Epoch 18][30/66] loss=0.924309 avg=0.855927 VRAM=39.5GiB | 6.2% done | ETA(epoch): 695s + [Epoch 18][40/66] loss=0.798712 avg=0.853523 VRAM=39.2GiB | 6.2% done | ETA(epoch): 501s + [Epoch 18][50/66] loss=0.850436 avg=0.858755 VRAM=39.5GiB | 6.3% done | ETA(epoch): 309s + [Epoch 18][60/66] loss=0.984573 avg=0.858960 VRAM=39.2GiB | 6.3% done | ETA(epoch): 116s + Train loss: 0.862053 (1275.8s) ETA: 6675min + Val loss: 0.896668 [t_0.0-0.2=1.0788 t_0.2-0.4=1.0067 t_0.4-0.6=0.8742 t_0.6-0.8=0.7410 t_0.8-1.0=0.7043] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 19/299 (6% done) --- + [Epoch 19][10/66] loss=0.924391 avg=0.886959 VRAM=39.5GiB | 6.4% done | ETA(epoch): 1086s + [Epoch 19][20/66] loss=0.962460 avg=0.882546 VRAM=39.2GiB | 6.4% done | ETA(epoch): 893s + [Epoch 19][30/66] loss=0.999629 avg=0.892306 VRAM=39.5GiB | 6.5% done | ETA(epoch): 703s + [Epoch 19][40/66] loss=0.879944 avg=0.887365 VRAM=39.2GiB | 6.5% done | ETA(epoch): 507s + [Epoch 19][50/66] loss=0.811275 avg=0.883916 VRAM=39.5GiB | 6.6% done | ETA(epoch): 312s + [Epoch 19][60/66] loss=0.982708 avg=0.877681 VRAM=39.2GiB | 6.6% done | ETA(epoch): 117s + Train loss: 0.879816 (1286.0s) ETA: 6641min + Val loss: 0.868786 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9979 t_0.4-0.6=0.8586 t_0.6-0.8=0.7319 t_0.8-1.0=0.7008] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 20/299 (7% done) --- + [Epoch 20][10/66] loss=0.856469 avg=0.863362 VRAM=39.5GiB | 6.7% done | ETA(epoch): 1086s + [Epoch 20][20/66] loss=0.822420 avg=0.849280 VRAM=39.2GiB | 6.8% done | ETA(epoch): 891s + [Epoch 20][30/66] loss=1.027833 avg=0.874630 VRAM=39.5GiB | 6.8% done | ETA(epoch): 696s + [Epoch 20][40/66] loss=0.827258 avg=0.873283 VRAM=39.2GiB | 6.9% done | ETA(epoch): 503s + [Epoch 20][50/66] loss=0.851559 avg=0.876463 VRAM=39.5GiB | 6.9% done | ETA(epoch): 310s + [Epoch 20][60/66] loss=0.910813 avg=0.881599 VRAM=39.2GiB | 7.0% done | ETA(epoch): 116s + Train loss: 0.880812 (1279.1s) ETA: 6607min + Val loss: 0.869345 [t_0.0-0.2=1.0868 t_0.2-0.4=1.0300 t_0.4-0.6=0.8551 t_0.6-0.8=0.7125 t_0.8-1.0=0.6997] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0020 +[MEM @ epoch 20 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 21/299 (7% done) --- + [Epoch 21][10/66] loss=0.754667 avg=0.876074 VRAM=39.5GiB | 7.1% done | ETA(epoch): 1088s + [Epoch 21][20/66] loss=0.927053 avg=0.878684 VRAM=39.2GiB | 7.1% done | ETA(epoch): 895s + [Epoch 21][30/66] loss=0.875945 avg=0.873816 VRAM=39.5GiB | 7.2% done | ETA(epoch): 701s + [Epoch 21][40/66] loss=0.815360 avg=0.867616 VRAM=39.2GiB | 7.2% done | ETA(epoch): 506s + [Epoch 21][50/66] loss=1.030356 avg=0.871594 VRAM=39.5GiB | 7.3% done | ETA(epoch): 311s + [Epoch 21][60/66] loss=0.865649 avg=0.873323 VRAM=39.2GiB | 7.3% done | ETA(epoch): 117s + Train loss: 0.872290 (1281.8s) ETA: 6574min + Val loss: 0.858287 [t_0.0-0.2=1.0874 t_0.2-0.4=0.9940 t_0.4-0.6=0.8300 t_0.6-0.8=0.7061 t_0.8-1.0=0.6968] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 22/299 (7% done) --- + [Epoch 22][10/66] loss=0.863476 avg=0.898809 VRAM=39.5GiB | 7.4% done | ETA(epoch): 1089s + [Epoch 22][20/66] loss=0.784709 avg=0.865443 VRAM=39.2GiB | 7.4% done | ETA(epoch): 892s + [Epoch 22][30/66] loss=0.840473 avg=0.861631 VRAM=39.5GiB | 7.5% done | ETA(epoch): 699s + [Epoch 22][40/66] loss=0.933998 avg=0.863247 VRAM=39.2GiB | 7.5% done | ETA(epoch): 505s + [Epoch 22][50/66] loss=0.857683 avg=0.858883 VRAM=39.5GiB | 7.6% done | ETA(epoch): 310s + [Epoch 22][60/66] loss=0.835798 avg=0.861873 VRAM=39.2GiB | 7.6% done | ETA(epoch): 116s + Train loss: 0.860047 (1280.1s) ETA: 6542min + Val loss: 0.867288 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0010 t_0.4-0.6=0.8682 t_0.6-0.8=0.7130 t_0.8-1.0=0.6862] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 23/299 (8% done) --- + [Epoch 23][10/66] loss=0.793285 avg=0.890521 VRAM=39.5GiB | 7.7% done | ETA(epoch): 1089s + [Epoch 23][20/66] loss=0.747044 avg=0.879972 VRAM=39.2GiB | 7.8% done | ETA(epoch): 896s + [Epoch 23][30/66] loss=0.923480 avg=0.880158 VRAM=39.5GiB | 7.8% done | ETA(epoch): 701s + [Epoch 23][40/66] loss=0.884969 avg=0.880951 VRAM=39.2GiB | 7.9% done | ETA(epoch): 506s + [Epoch 23][50/66] loss=0.930435 avg=0.880355 VRAM=39.5GiB | 7.9% done | ETA(epoch): 311s + [Epoch 23][60/66] loss=0.948033 avg=0.876959 VRAM=39.2GiB | 8.0% done | ETA(epoch): 117s + Train loss: 0.874571 (1286.7s) ETA: 6511min + Val loss: 0.876350 [t_0.0-0.2=1.0788 t_0.2-0.4=1.0182 t_0.4-0.6=0.8565 t_0.6-0.8=0.7395 t_0.8-1.0=0.6734] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 24/299 (8% done) --- + [Epoch 24][10/66] loss=0.896951 avg=0.834469 VRAM=39.5GiB | 8.1% done | ETA(epoch): 1085s + [Epoch 24][20/66] loss=0.927482 avg=0.843477 VRAM=39.2GiB | 8.1% done | ETA(epoch): 889s + [Epoch 24][30/66] loss=0.832705 avg=0.852319 VRAM=39.5GiB | 8.2% done | ETA(epoch): 695s + [Epoch 24][40/66] loss=0.894989 avg=0.848970 VRAM=39.2GiB | 8.2% done | ETA(epoch): 502s + [Epoch 24][50/66] loss=0.845766 avg=0.849455 VRAM=39.5GiB | 8.3% done | ETA(epoch): 309s + [Epoch 24][60/66] loss=0.880314 avg=0.853331 VRAM=39.2GiB | 8.3% done | ETA(epoch): 116s + Train loss: 0.856851 (1277.9s) ETA: 6480min + Val loss: 0.885633 [t_0.0-0.2=1.0798 t_0.2-0.4=0.9875 t_0.4-0.6=0.8363 t_0.6-0.8=0.7560 t_0.8-1.0=0.6753] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 25/299 (8% done) --- + [Epoch 25][10/66] loss=0.806197 avg=0.866902 VRAM=39.5GiB | 8.4% done | ETA(epoch): 1083s + [Epoch 25][20/66] loss=0.966920 avg=0.885849 VRAM=39.2GiB | 8.4% done | ETA(epoch): 889s + [Epoch 25][30/66] loss=0.853666 avg=0.878951 VRAM=39.5GiB | 8.5% done | ETA(epoch): 696s + [Epoch 25][40/66] loss=0.793069 avg=0.869438 VRAM=39.2GiB | 8.5% done | ETA(epoch): 505s + [Epoch 25][50/66] loss=0.828628 avg=0.868790 VRAM=39.5GiB | 8.6% done | ETA(epoch): 311s + [Epoch 25][60/66] loss=0.862440 avg=0.865839 VRAM=39.2GiB | 8.6% done | ETA(epoch): 117s + Train loss: 0.866804 (1283.5s) ETA: 6451min + Val loss: 0.868660 [t_0.0-0.2=1.0871 t_0.2-0.4=1.0250 t_0.4-0.6=0.9025 t_0.6-0.8=0.7144 t_0.8-1.0=0.6860] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 26/299 (9% done) --- + [Epoch 26][10/66] loss=0.870744 avg=0.877630 VRAM=39.5GiB | 8.7% done | ETA(epoch): 1088s + [Epoch 26][20/66] loss=0.833552 avg=0.874415 VRAM=39.2GiB | 8.8% done | ETA(epoch): 891s + [Epoch 26][30/66] loss=0.952268 avg=0.876529 VRAM=39.5GiB | 8.8% done | ETA(epoch): 696s + [Epoch 26][40/66] loss=0.906307 avg=0.873729 VRAM=39.2GiB | 8.9% done | ETA(epoch): 502s + [Epoch 26][50/66] loss=0.815136 avg=0.872997 VRAM=39.5GiB | 8.9% done | ETA(epoch): 309s + [Epoch 26][60/66] loss=0.963376 avg=0.874769 VRAM=39.2GiB | 9.0% done | ETA(epoch): 116s + Train loss: 0.873047 (1275.6s) ETA: 6420min + Val loss: 0.818951 [t_0.0-0.2=1.0804 t_0.2-0.4=1.0454 t_0.4-0.6=0.8564 t_0.6-0.8=0.7189 t_0.8-1.0=0.6780] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0026 (BEST) + Deleted old checkpoint: checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 27/299 (9% done) --- + [Epoch 27][10/66] loss=0.887320 avg=0.851941 VRAM=39.5GiB | 9.1% done | ETA(epoch): 1086s + [Epoch 27][20/66] loss=0.799648 avg=0.843723 VRAM=39.2GiB | 9.1% done | ETA(epoch): 893s + [Epoch 27][30/66] loss=0.836701 avg=0.842536 VRAM=39.5GiB | 9.2% done | ETA(epoch): 699s + [Epoch 27][40/66] loss=0.866172 avg=0.852454 VRAM=39.2GiB | 9.2% done | ETA(epoch): 504s + [Epoch 27][50/66] loss=0.782942 avg=0.856447 VRAM=39.5GiB | 9.3% done | ETA(epoch): 310s + [Epoch 27][60/66] loss=1.048570 avg=0.867327 VRAM=39.2GiB | 9.3% done | ETA(epoch): 116s + Train loss: 0.869992 (1277.1s) ETA: 6391min + Val loss: 0.881880 [t_0.0-0.2=1.0588 t_0.2-0.4=0.9849 t_0.4-0.6=0.8748 t_0.6-0.8=0.7327 t_0.8-1.0=0.7122] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 28/299 (9% done) --- + [Epoch 28][10/66] loss=0.945661 avg=0.879544 VRAM=39.5GiB | 9.4% done | ETA(epoch): 1088s + [Epoch 28][20/66] loss=0.851032 avg=0.884338 VRAM=39.2GiB | 9.4% done | ETA(epoch): 898s + [Epoch 28][30/66] loss=0.955285 avg=0.883214 VRAM=39.5GiB | 9.5% done | ETA(epoch): 702s + [Epoch 28][40/66] loss=0.977638 avg=0.876542 VRAM=39.2GiB | 9.5% done | ETA(epoch): 506s + [Epoch 28][50/66] loss=0.829134 avg=0.877205 VRAM=39.5GiB | 9.6% done | ETA(epoch): 311s + [Epoch 28][60/66] loss=0.874830 avg=0.878034 VRAM=39.2GiB | 9.6% done | ETA(epoch): 117s + Train loss: 0.876755 (1281.3s) ETA: 6362min + Val loss: 0.885780 [t_0.0-0.2=1.0666 t_0.2-0.4=1.0224 t_0.4-0.6=0.8401 t_0.6-0.8=0.7413 t_0.8-1.0=0.6892] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 29/299 (10% done) --- + [Epoch 29][10/66] loss=0.951021 avg=0.889294 VRAM=39.5GiB | 9.7% done | ETA(epoch): 1088s + [Epoch 29][20/66] loss=0.882264 avg=0.886876 VRAM=39.2GiB | 9.8% done | ETA(epoch): 895s + [Epoch 29][30/66] loss=0.922705 avg=0.881692 VRAM=39.5GiB | 9.8% done | ETA(epoch): 701s + [Epoch 29][40/66] loss=0.804053 avg=0.886214 VRAM=39.2GiB | 9.9% done | ETA(epoch): 506s + [Epoch 29][50/66] loss=0.950425 avg=0.891992 VRAM=39.5GiB | 9.9% done | ETA(epoch): 311s + [Epoch 29][60/66] loss=0.873258 avg=0.891898 VRAM=39.2GiB | 10.0% done | ETA(epoch): 117s + Train loss: 0.886614 (1282.5s) ETA: 6334min + Val loss: 0.863268 [t_0.0-0.2=1.0819 t_0.2-0.4=0.9593 t_0.4-0.6=0.8519 t_0.6-0.8=0.7311 t_0.8-1.0=0.6691] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0029 +[MEM @ epoch 29 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 30/299 (10% done) --- + [MilestoneVis] train_0 step 1980 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 1980 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 1980 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 1980 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 1980 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 1980 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 1980 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 1980 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 1980 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 1980 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + [Epoch 30][10/66] loss=0.837436 avg=0.861457 VRAM=39.5GiB | 10.1% done | ETA(epoch): 6792s + [Epoch 30][20/66] loss=0.895585 avg=0.867036 VRAM=39.2GiB | 10.1% done | ETA(epoch): 3234s + [Epoch 30][30/66] loss=0.811797 avg=0.856760 VRAM=39.5GiB | 10.2% done | ETA(epoch): 1919s + [Epoch 30][40/66] loss=0.812369 avg=0.868493 VRAM=39.2GiB | 10.2% done | ETA(epoch): 1166s + [Epoch 30][50/66] loss=0.840086 avg=0.867046 VRAM=39.5GiB | 10.3% done | ETA(epoch): 636s + [Epoch 30][60/66] loss=0.912497 avg=0.876657 VRAM=39.2GiB | 10.3% done | ETA(epoch): 218s + Train loss: 0.873778 (2298.9s) ETA: 6453min + Val loss: 0.877363 [t_0.0-0.2=1.0491 t_0.2-0.4=1.0035 t_0.4-0.6=0.8604 t_0.6-0.8=0.7532 t_0.8-1.0=0.6961] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 31/299 (10% done) --- + [Epoch 31][10/66] loss=0.788176 avg=0.813048 VRAM=39.5GiB | 10.4% done | ETA(epoch): 1085s + [Epoch 31][20/66] loss=0.895672 avg=0.837451 VRAM=39.2GiB | 10.4% done | ETA(epoch): 893s + [Epoch 31][30/66] loss=1.006506 avg=0.850751 VRAM=39.5GiB | 10.5% done | ETA(epoch): 700s + [Epoch 31][40/66] loss=0.747245 avg=0.855164 VRAM=39.2GiB | 10.5% done | ETA(epoch): 505s + [Epoch 31][50/66] loss=0.717613 avg=0.848745 VRAM=39.5GiB | 10.6% done | ETA(epoch): 311s + [Epoch 31][60/66] loss=0.899991 avg=0.856760 VRAM=39.2GiB | 10.6% done | ETA(epoch): 117s + Train loss: 0.860793 (1281.2s) ETA: 6420min + Val loss: 0.896673 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9932 t_0.4-0.6=0.8751 t_0.6-0.8=0.7247 t_0.8-1.0=0.6812] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 32/299 (11% done) --- + [Epoch 32][10/66] loss=0.918091 avg=0.875322 VRAM=39.5GiB | 10.7% done | ETA(epoch): 1087s + [Epoch 32][20/66] loss=0.854767 avg=0.862932 VRAM=39.2GiB | 10.8% done | ETA(epoch): 892s + [Epoch 32][30/66] loss=0.878001 avg=0.857526 VRAM=39.5GiB | 10.8% done | ETA(epoch): 697s + [Epoch 32][40/66] loss=1.019892 avg=0.862775 VRAM=39.2GiB | 10.9% done | ETA(epoch): 504s + [Epoch 32][50/66] loss=0.826766 avg=0.867102 VRAM=39.5GiB | 10.9% done | ETA(epoch): 310s + [Epoch 32][60/66] loss=0.972900 avg=0.870210 VRAM=39.2GiB | 11.0% done | ETA(epoch): 116s + Train loss: 0.869471 (1278.8s) ETA: 6388min + Val loss: 0.882020 [t_0.0-0.2=1.0813 t_0.2-0.4=0.9698 t_0.4-0.6=0.8752 t_0.6-0.8=0.7286 t_0.8-1.0=0.6786] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 33/299 (11% done) --- + [Epoch 33][10/66] loss=0.783430 avg=0.858334 VRAM=39.5GiB | 11.1% done | ETA(epoch): 1086s + [Epoch 33][20/66] loss=1.016717 avg=0.878339 VRAM=39.2GiB | 11.1% done | ETA(epoch): 892s + [Epoch 33][30/66] loss=0.815479 avg=0.880345 VRAM=39.5GiB | 11.2% done | ETA(epoch): 697s + [Epoch 33][40/66] loss=0.758517 avg=0.872464 VRAM=39.2GiB | 11.2% done | ETA(epoch): 505s + [Epoch 33][50/66] loss=0.885449 avg=0.866877 VRAM=39.5GiB | 11.3% done | ETA(epoch): 311s + [Epoch 33][60/66] loss=0.799750 avg=0.864820 VRAM=39.2GiB | 11.3% done | ETA(epoch): 117s + Train loss: 0.866059 (1280.8s) ETA: 6356min + Val loss: 0.837718 [t_0.0-0.2=1.0553 t_0.2-0.4=0.9973 t_0.4-0.6=0.8484 t_0.6-0.8=0.7256 t_0.8-1.0=0.6893] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 34/299 (11% done) --- + [Epoch 34][10/66] loss=0.865584 avg=0.830149 VRAM=39.5GiB | 11.4% done | ETA(epoch): 1086s + [Epoch 34][20/66] loss=0.822243 avg=0.866980 VRAM=39.2GiB | 11.4% done | ETA(epoch): 894s + [Epoch 34][30/66] loss=0.823335 avg=0.865783 VRAM=39.5GiB | 11.5% done | ETA(epoch): 698s + [Epoch 34][40/66] loss=0.949161 avg=0.872516 VRAM=39.2GiB | 11.5% done | ETA(epoch): 504s + [Epoch 34][50/66] loss=0.929149 avg=0.867088 VRAM=39.5GiB | 11.6% done | ETA(epoch): 310s + [Epoch 34][60/66] loss=0.929874 avg=0.867852 VRAM=39.2GiB | 11.6% done | ETA(epoch): 116s + Train loss: 0.865055 (1277.9s) ETA: 6325min + Val loss: 0.856192 [t_0.0-0.2=1.0669 t_0.2-0.4=1.0035 t_0.4-0.6=0.8354 t_0.6-0.8=0.7150 t_0.8-1.0=0.6742] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 35/299 (12% done) --- + [Epoch 35][10/66] loss=0.926938 avg=0.890938 VRAM=39.5GiB | 11.7% done | ETA(epoch): 1088s + [Epoch 35][20/66] loss=0.829560 avg=0.883410 VRAM=39.2GiB | 11.8% done | ETA(epoch): 895s + [Epoch 35][30/66] loss=0.749938 avg=0.882009 VRAM=39.5GiB | 11.8% done | ETA(epoch): 700s + [Epoch 35][40/66] loss=0.908548 avg=0.878103 VRAM=39.2GiB | 11.9% done | ETA(epoch): 506s + [Epoch 35][50/66] loss=0.836636 avg=0.874467 VRAM=39.5GiB | 11.9% done | ETA(epoch): 311s + [Epoch 35][60/66] loss=0.800338 avg=0.865281 VRAM=39.2GiB | 12.0% done | ETA(epoch): 117s + Train loss: 0.867562 (1283.6s) ETA: 6294min + Val loss: 0.856825 [t_0.0-0.2=1.0902 t_0.2-0.4=1.0096 t_0.4-0.6=0.8537 t_0.6-0.8=0.7308 t_0.8-1.0=0.6945] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 17.1/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 36/299 (12% done) --- + [Epoch 36][10/66] loss=0.934806 avg=0.876946 VRAM=39.5GiB | 12.1% done | ETA(epoch): 1089s + [Epoch 36][20/66] loss=0.758545 avg=0.865725 VRAM=39.2GiB | 12.1% done | ETA(epoch): 896s + [Epoch 36][30/66] loss=0.985214 avg=0.861796 VRAM=39.5GiB | 12.2% done | ETA(epoch): 701s + [Epoch 36][40/66] loss=0.917302 avg=0.873364 VRAM=39.2GiB | 12.2% done | ETA(epoch): 506s + [Epoch 36][50/66] loss=0.801886 avg=0.874316 VRAM=39.5GiB | 12.3% done | ETA(epoch): 311s + [Epoch 36][60/66] loss=0.795551 avg=0.874109 VRAM=39.2GiB | 12.3% done | ETA(epoch): 116s + Train loss: 0.870616 (1279.2s) ETA: 6264min + Val loss: 0.843780 [t_0.0-0.2=1.0623 t_0.2-0.4=1.0026 t_0.4-0.6=0.8078 t_0.6-0.8=0.7378 t_0.8-1.0=0.6714] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 17.1/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 37/299 (12% done) --- + [Epoch 37][10/66] loss=0.944333 avg=0.865489 VRAM=39.5GiB | 12.4% done | ETA(epoch): 1081s + [Epoch 37][20/66] loss=0.841136 avg=0.855948 VRAM=39.2GiB | 12.4% done | ETA(epoch): 891s + [Epoch 37][30/66] loss=0.775187 avg=0.853827 VRAM=39.5GiB | 12.5% done | ETA(epoch): 697s + [Epoch 37][40/66] loss=0.888989 avg=0.849257 VRAM=39.2GiB | 12.5% done | ETA(epoch): 503s + [Epoch 37][50/66] loss=0.890653 avg=0.847763 VRAM=39.5GiB | 12.6% done | ETA(epoch): 309s + [Epoch 37][60/66] loss=1.026871 avg=0.852789 VRAM=39.2GiB | 12.6% done | ETA(epoch): 116s + Train loss: 0.856356 (1277.1s) ETA: 6233min + Val loss: 0.845275 [t_0.0-0.2=1.0922 t_0.2-0.4=0.9081 t_0.4-0.6=0.9049 t_0.6-0.8=0.7250 t_0.8-1.0=0.6838] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 17.1/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 38/299 (13% done) --- + [Epoch 38][10/66] loss=0.785674 avg=0.816728 VRAM=39.5GiB | 12.7% done | ETA(epoch): 1088s + [Epoch 38][20/66] loss=0.940983 avg=0.848575 VRAM=39.2GiB | 12.8% done | ETA(epoch): 894s + [Epoch 38][30/66] loss=0.750419 avg=0.850198 VRAM=39.5GiB | 12.8% done | ETA(epoch): 699s + [Epoch 38][40/66] loss=0.971265 avg=0.855995 VRAM=39.2GiB | 12.9% done | ETA(epoch): 505s + [Epoch 38][50/66] loss=0.839363 avg=0.853113 VRAM=39.5GiB | 12.9% done | ETA(epoch): 310s + [Epoch 38][60/66] loss=0.923922 avg=0.853976 VRAM=39.2GiB | 13.0% done | ETA(epoch): 116s + Train loss: 0.853972 (1279.4s) ETA: 6204min + Val loss: 0.893461 [t_0.0-0.2=1.0800 t_0.2-0.4=0.9880 t_0.4-0.6=0.8613 t_0.6-0.8=0.7281 t_0.8-1.0=0.6895] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 39/299 (13% done) --- + [Epoch 39][10/66] loss=0.807687 avg=0.863116 VRAM=39.5GiB | 13.1% done | ETA(epoch): 1086s + [Epoch 39][20/66] loss=0.850585 avg=0.870242 VRAM=39.2GiB | 13.1% done | ETA(epoch): 892s + [Epoch 39][30/66] loss=0.830186 avg=0.858879 VRAM=39.5GiB | 13.2% done | ETA(epoch): 697s + [Epoch 39][40/66] loss=0.843384 avg=0.855844 VRAM=39.2GiB | 13.2% done | ETA(epoch): 503s + [Epoch 39][50/66] loss=0.846277 avg=0.854318 VRAM=39.5GiB | 13.3% done | ETA(epoch): 309s + [Epoch 39][60/66] loss=0.773204 avg=0.855383 VRAM=39.2GiB | 13.3% done | ETA(epoch): 116s + Train loss: 0.855974 (1275.1s) ETA: 6174min + Val loss: 0.858698 [t_0.0-0.2=1.0706 t_0.2-0.4=0.9908 t_0.4-0.6=0.8418 t_0.6-0.8=0.7411 t_0.8-1.0=0.6705] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 17.1/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 40/299 (13% done) --- + [Epoch 40][10/66] loss=0.807870 avg=0.874592 VRAM=39.5GiB | 13.4% done | ETA(epoch): 1087s + [Epoch 40][20/66] loss=0.868684 avg=0.882363 VRAM=39.2GiB | 13.4% done | ETA(epoch): 894s + [Epoch 40][30/66] loss=0.840027 avg=0.866670 VRAM=39.5GiB | 13.5% done | ETA(epoch): 698s + [Epoch 40][40/66] loss=0.935916 avg=0.870382 VRAM=39.2GiB | 13.5% done | ETA(epoch): 505s + [Epoch 40][50/66] loss=0.843153 avg=0.875004 VRAM=39.5GiB | 13.6% done | ETA(epoch): 311s + [Epoch 40][60/66] loss=0.834222 avg=0.875814 VRAM=39.2GiB | 13.6% done | ETA(epoch): 117s + Train loss: 0.877161 (1283.2s) ETA: 6145min + Val loss: 0.866852 [t_0.0-0.2=1.0764 t_0.2-0.4=0.9938 t_0.4-0.6=0.8572 t_0.6-0.8=0.7195 t_0.8-1.0=0.6744] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 17.2/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 41/299 (14% done) --- + [Epoch 41][10/66] loss=0.958659 avg=0.901873 VRAM=39.5GiB | 13.7% done | ETA(epoch): 1085s + [Epoch 41][20/66] loss=0.913630 avg=0.884762 VRAM=39.2GiB | 13.8% done | ETA(epoch): 890s + [Epoch 41][30/66] loss=0.849259 avg=0.870698 VRAM=39.5GiB | 13.8% done | ETA(epoch): 696s + [Epoch 41][40/66] loss=0.947915 avg=0.869335 VRAM=39.2GiB | 13.9% done | ETA(epoch): 504s + [Epoch 41][50/66] loss=0.892945 avg=0.862595 VRAM=39.5GiB | 13.9% done | ETA(epoch): 310s + [Epoch 41][60/66] loss=0.894809 avg=0.859602 VRAM=39.2GiB | 14.0% done | ETA(epoch): 116s + Train loss: 0.857571 (1277.6s) ETA: 6116min + Val loss: 0.854791 [t_0.0-0.2=1.0779 t_0.2-0.4=1.0078 t_0.4-0.6=0.8254 t_0.6-0.8=0.6992 t_0.8-1.0=0.6622] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 17.1/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 42/299 (14% done) --- + [Epoch 42][10/66] loss=0.864506 avg=0.860039 VRAM=39.5GiB | 14.1% done | ETA(epoch): 1084s + [Epoch 42][20/66] loss=0.974894 avg=0.867399 VRAM=39.2GiB | 14.1% done | ETA(epoch): 889s + [Epoch 42][30/66] loss=0.854161 avg=0.872206 VRAM=39.5GiB | 14.2% done | ETA(epoch): 695s + [Epoch 42][40/66] loss=0.884880 avg=0.867714 VRAM=39.2GiB | 14.2% done | ETA(epoch): 503s + [Epoch 42][50/66] loss=0.858961 avg=0.857783 VRAM=39.5GiB | 14.3% done | ETA(epoch): 309s + [Epoch 42][60/66] loss=0.840076 avg=0.853685 VRAM=39.2GiB | 14.3% done | ETA(epoch): 116s + Train loss: 0.855712 (1275.5s) ETA: 6088min + Val loss: 0.842251 [t_0.0-0.2=1.0679 t_0.2-0.4=1.0171 t_0.4-0.6=0.8042 t_0.6-0.8=0.7368 t_0.8-1.0=0.6705] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 43/299 (14% done) --- + [Epoch 43][10/66] loss=0.951320 avg=0.868753 VRAM=39.5GiB | 14.4% done | ETA(epoch): 1087s + [Epoch 43][20/66] loss=0.856955 avg=0.873814 VRAM=39.2GiB | 14.4% done | ETA(epoch): 899s + [Epoch 43][30/66] loss=0.857716 avg=0.859206 VRAM=39.5GiB | 14.5% done | ETA(epoch): 701s + [Epoch 43][40/66] loss=0.822525 avg=0.859127 VRAM=39.2GiB | 14.5% done | ETA(epoch): 505s + [Epoch 43][50/66] loss=0.852270 avg=0.861826 VRAM=39.5GiB | 14.6% done | ETA(epoch): 311s + [Epoch 43][60/66] loss=0.947795 avg=0.863562 VRAM=39.2GiB | 14.6% done | ETA(epoch): 116s + Train loss: 0.861236 (1279.1s) ETA: 6059min + Val loss: 0.827336 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9952 t_0.4-0.6=0.8274 t_0.6-0.8=0.7140 t_0.8-1.0=0.6641] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 17.2/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 44/299 (15% done) --- + [Epoch 44][10/66] loss=0.885670 avg=0.863685 VRAM=39.5GiB | 14.7% done | ETA(epoch): 1096s + [Epoch 44][20/66] loss=0.820325 avg=0.860331 VRAM=39.2GiB | 14.8% done | ETA(epoch): 898s + [Epoch 44][30/66] loss=0.993959 avg=0.855601 VRAM=39.5GiB | 14.8% done | ETA(epoch): 702s + [Epoch 44][40/66] loss=1.026708 avg=0.857005 VRAM=39.2GiB | 14.9% done | ETA(epoch): 506s + [Epoch 44][50/66] loss=0.823775 avg=0.851693 VRAM=39.5GiB | 14.9% done | ETA(epoch): 311s + [Epoch 44][60/66] loss=0.910302 avg=0.852923 VRAM=39.2GiB | 15.0% done | ETA(epoch): 116s + Train loss: 0.853263 (1280.4s) ETA: 6032min + Val loss: 0.871767 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9936 t_0.4-0.6=0.8778 t_0.6-0.8=0.7027 t_0.8-1.0=0.6750] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 45/299 (15% done) --- + [Epoch 45][10/66] loss=0.802213 avg=0.857772 VRAM=39.5GiB | 15.1% done | ETA(epoch): 1087s + [Epoch 45][20/66] loss=0.861209 avg=0.847339 VRAM=39.2GiB | 15.1% done | ETA(epoch): 891s + [Epoch 45][30/66] loss=0.871952 avg=0.860605 VRAM=39.5GiB | 15.2% done | ETA(epoch): 697s + [Epoch 45][40/66] loss=0.831980 avg=0.856404 VRAM=39.2GiB | 15.2% done | ETA(epoch): 503s + [Epoch 45][50/66] loss=0.859498 avg=0.855138 VRAM=39.5GiB | 15.3% done | ETA(epoch): 309s + [Epoch 45][60/66] loss=0.756129 avg=0.858585 VRAM=39.2GiB | 15.3% done | ETA(epoch): 117s + Train loss: 0.855285 (1281.7s) ETA: 6004min + Val loss: 0.848480 [t_0.0-0.2=1.0670 t_0.2-0.4=0.9671 t_0.4-0.6=0.8379 t_0.6-0.8=0.7142 t_0.8-1.0=0.6533] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 46/299 (15% done) --- + [Epoch 46][10/66] loss=0.734687 avg=0.888482 VRAM=39.5GiB | 15.4% done | ETA(epoch): 1087s + [Epoch 46][20/66] loss=0.891047 avg=0.883220 VRAM=39.2GiB | 15.4% done | ETA(epoch): 891s + [Epoch 46][30/66] loss=0.955267 avg=0.883854 VRAM=39.5GiB | 15.5% done | ETA(epoch): 696s + [Epoch 46][40/66] loss=0.909161 avg=0.884061 VRAM=39.2GiB | 15.5% done | ETA(epoch): 503s + [Epoch 46][50/66] loss=0.825037 avg=0.881216 VRAM=39.5GiB | 15.6% done | ETA(epoch): 309s + [Epoch 46][60/66] loss=0.940727 avg=0.877236 VRAM=39.2GiB | 15.6% done | ETA(epoch): 116s + Train loss: 0.873308 (1275.9s) ETA: 5976min + Val loss: 0.878547 [t_0.0-0.2=1.0707 t_0.2-0.4=1.0050 t_0.4-0.6=0.8137 t_0.6-0.8=0.7579 t_0.8-1.0=0.6605] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 47/299 (16% done) --- + [Epoch 47][10/66] loss=0.833338 avg=0.842265 VRAM=39.5GiB | 15.7% done | ETA(epoch): 1083s + [Epoch 47][20/66] loss=0.829039 avg=0.837906 VRAM=39.2GiB | 15.8% done | ETA(epoch): 889s + [Epoch 47][30/66] loss=0.939044 avg=0.849004 VRAM=39.5GiB | 15.8% done | ETA(epoch): 695s + [Epoch 47][40/66] loss=0.803696 avg=0.856389 VRAM=39.2GiB | 15.9% done | ETA(epoch): 502s + [Epoch 47][50/66] loss=0.924086 avg=0.854923 VRAM=39.5GiB | 15.9% done | ETA(epoch): 310s + [Epoch 47][60/66] loss=0.951595 avg=0.856091 VRAM=39.2GiB | 16.0% done | ETA(epoch): 116s + Train loss: 0.861744 (1278.9s) ETA: 5949min + Val loss: 0.856283 [t_0.0-0.2=1.0615 t_0.2-0.4=1.0081 t_0.4-0.6=0.8545 t_0.6-0.8=0.7322 t_0.8-1.0=0.6665] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 48/299 (16% done) --- + [Epoch 48][10/66] loss=0.852034 avg=0.824180 VRAM=39.5GiB | 16.1% done | ETA(epoch): 1088s + [Epoch 48][20/66] loss=0.987399 avg=0.854664 VRAM=39.2GiB | 16.1% done | ETA(epoch): 892s + [Epoch 48][30/66] loss=0.858325 avg=0.845830 VRAM=39.5GiB | 16.2% done | ETA(epoch): 697s + [Epoch 48][40/66] loss=0.797950 avg=0.844742 VRAM=39.2GiB | 16.2% done | ETA(epoch): 504s + [Epoch 48][50/66] loss=0.762718 avg=0.842893 VRAM=39.5GiB | 16.3% done | ETA(epoch): 310s + [Epoch 48][60/66] loss=0.875738 avg=0.848952 VRAM=39.2GiB | 16.3% done | ETA(epoch): 116s + Train loss: 0.846635 (1277.6s) ETA: 5921min + Val loss: 0.864315 [t_0.0-0.2=1.0621 t_0.2-0.4=1.0139 t_0.4-0.6=0.8429 t_0.6-0.8=0.7271 t_0.8-1.0=0.6724] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 49/299 (16% done) --- + [Epoch 49][10/66] loss=0.896213 avg=0.877296 VRAM=39.5GiB | 16.4% done | ETA(epoch): 1082s + [Epoch 49][20/66] loss=0.813667 avg=0.885210 VRAM=39.2GiB | 16.4% done | ETA(epoch): 888s + [Epoch 49][30/66] loss=0.964786 avg=0.877327 VRAM=39.5GiB | 16.5% done | ETA(epoch): 698s + [Epoch 49][40/66] loss=0.879789 avg=0.877640 VRAM=39.2GiB | 16.5% done | ETA(epoch): 504s + [Epoch 49][50/66] loss=0.755960 avg=0.872732 VRAM=39.5GiB | 16.6% done | ETA(epoch): 310s + [Epoch 49][60/66] loss=0.934253 avg=0.869672 VRAM=39.2GiB | 16.6% done | ETA(epoch): 116s + Train loss: 0.869006 (1279.8s) ETA: 5894min + Val loss: 0.881906 [t_0.0-0.2=1.0702 t_0.2-0.4=0.9866 t_0.4-0.6=0.8767 t_0.6-0.8=0.7109 t_0.8-1.0=0.6675] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 50/299 (17% done) --- + [Epoch 50][10/66] loss=0.935053 avg=0.869786 VRAM=39.5GiB | 16.7% done | ETA(epoch): 1081s + [Epoch 50][20/66] loss=0.959422 avg=0.872866 VRAM=39.2GiB | 16.8% done | ETA(epoch): 900s + [Epoch 50][30/66] loss=0.930671 avg=0.873817 VRAM=39.5GiB | 16.8% done | ETA(epoch): 702s + [Epoch 50][40/66] loss=0.843995 avg=0.873597 VRAM=39.2GiB | 16.9% done | ETA(epoch): 506s + [Epoch 50][50/66] loss=0.736578 avg=0.869616 VRAM=39.5GiB | 16.9% done | ETA(epoch): 311s + [Epoch 50][60/66] loss=0.801087 avg=0.863808 VRAM=39.2GiB | 17.0% done | ETA(epoch): 117s + Train loss: 0.861668 (1280.8s) ETA: 5868min + Val loss: 0.880034 [t_0.0-0.2=1.0846 t_0.2-0.4=0.9815 t_0.4-0.6=0.8807 t_0.6-0.8=0.7269 t_0.8-1.0=0.6731] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 51/299 (17% done) --- + [Epoch 51][10/66] loss=0.728607 avg=0.867517 VRAM=39.5GiB | 17.1% done | ETA(epoch): 1097s + [Epoch 51][20/66] loss=0.749895 avg=0.863599 VRAM=39.2GiB | 17.1% done | ETA(epoch): 894s + [Epoch 51][30/66] loss=0.761057 avg=0.853465 VRAM=39.5GiB | 17.2% done | ETA(epoch): 698s + [Epoch 51][40/66] loss=0.793137 avg=0.852275 VRAM=39.2GiB | 17.2% done | ETA(epoch): 504s + [Epoch 51][50/66] loss=0.853010 avg=0.851817 VRAM=39.5GiB | 17.3% done | ETA(epoch): 310s + [Epoch 51][60/66] loss=0.901902 avg=0.847504 VRAM=39.2GiB | 17.3% done | ETA(epoch): 116s + Train loss: 0.851110 (1277.5s) ETA: 5841min + Val loss: 0.847410 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9777 t_0.4-0.6=0.8749 t_0.6-0.8=0.6981 t_0.8-1.0=0.6500] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 52/299 (17% done) --- + [Epoch 52][10/66] loss=0.952200 avg=0.841079 VRAM=39.5GiB | 17.4% done | ETA(epoch): 1084s + [Epoch 52][20/66] loss=0.943584 avg=0.839548 VRAM=39.2GiB | 17.4% done | ETA(epoch): 889s + [Epoch 52][30/66] loss=0.794292 avg=0.849777 VRAM=39.5GiB | 17.5% done | ETA(epoch): 695s + [Epoch 52][40/66] loss=0.809745 avg=0.842258 VRAM=39.2GiB | 17.5% done | ETA(epoch): 502s + [Epoch 52][50/66] loss=0.779936 avg=0.841535 VRAM=39.5GiB | 17.6% done | ETA(epoch): 309s + [Epoch 52][60/66] loss=0.830129 avg=0.841074 VRAM=39.2GiB | 17.6% done | ETA(epoch): 116s + Train loss: 0.836695 (1280.5s) ETA: 5815min + Val loss: 0.863836 [t_0.0-0.2=1.0899 t_0.2-0.4=0.9729 t_0.4-0.6=0.8538 t_0.6-0.8=0.7179 t_0.8-1.0=0.6676] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 53/299 (18% done) --- + [Epoch 53][10/66] loss=0.744597 avg=0.830828 VRAM=39.5GiB | 17.7% done | ETA(epoch): 1087s + [Epoch 53][20/66] loss=0.834828 avg=0.836037 VRAM=39.2GiB | 17.8% done | ETA(epoch): 891s + [Epoch 53][30/66] loss=0.849691 avg=0.847671 VRAM=39.5GiB | 17.8% done | ETA(epoch): 696s + [Epoch 53][40/66] loss=0.807319 avg=0.849332 VRAM=39.2GiB | 17.9% done | ETA(epoch): 503s + [Epoch 53][50/66] loss=0.813482 avg=0.848528 VRAM=39.5GiB | 17.9% done | ETA(epoch): 309s + [Epoch 53][60/66] loss=0.749310 avg=0.858093 VRAM=39.2GiB | 18.0% done | ETA(epoch): 116s + Train loss: 0.859718 (1275.6s) ETA: 5788min + Val loss: 0.866131 [t_0.0-0.2=1.0789 t_0.2-0.4=1.0008 t_0.4-0.6=0.8278 t_0.6-0.8=0.6890 t_0.8-1.0=0.6558] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 54/299 (18% done) --- + [Epoch 54][10/66] loss=0.885181 avg=0.839589 VRAM=39.5GiB | 18.1% done | ETA(epoch): 1084s + [Epoch 54][20/66] loss=0.792803 avg=0.848647 VRAM=39.2GiB | 18.1% done | ETA(epoch): 890s + [Epoch 54][30/66] loss=0.808840 avg=0.843036 VRAM=39.5GiB | 18.2% done | ETA(epoch): 697s + [Epoch 54][40/66] loss=0.901357 avg=0.842760 VRAM=39.2GiB | 18.2% done | ETA(epoch): 504s + [Epoch 54][50/66] loss=0.861798 avg=0.848874 VRAM=39.5GiB | 18.3% done | ETA(epoch): 310s + [Epoch 54][60/66] loss=0.921773 avg=0.850277 VRAM=39.2GiB | 18.3% done | ETA(epoch): 116s + Train loss: 0.851618 (1277.7s) ETA: 5761min + Val loss: 0.863888 [t_0.0-0.2=1.0821 t_0.2-0.4=0.9538 t_0.4-0.6=0.8230 t_0.6-0.8=0.7438 t_0.8-1.0=0.6539] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 55/299 (18% done) --- + [Epoch 55][10/66] loss=0.955586 avg=0.865016 VRAM=39.5GiB | 18.4% done | ETA(epoch): 1087s + [Epoch 55][20/66] loss=0.868692 avg=0.872994 VRAM=39.2GiB | 18.4% done | ETA(epoch): 892s + [Epoch 55][30/66] loss=0.848626 avg=0.872221 VRAM=39.5GiB | 18.5% done | ETA(epoch): 697s + [Epoch 55][40/66] loss=0.802660 avg=0.865102 VRAM=39.2GiB | 18.5% done | ETA(epoch): 504s + [Epoch 55][50/66] loss=0.917953 avg=0.868261 VRAM=39.5GiB | 18.6% done | ETA(epoch): 310s + [Epoch 55][60/66] loss=0.883548 avg=0.866279 VRAM=39.2GiB | 18.6% done | ETA(epoch): 116s + Train loss: 0.864009 (1280.0s) ETA: 5735min + Val loss: 0.839248 [t_0.0-0.2=1.0570 t_0.2-0.4=1.0433 t_0.4-0.6=0.8542 t_0.6-0.8=0.7245 t_0.8-1.0=0.6579] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 56/299 (19% done) --- + [Epoch 56][10/66] loss=0.756443 avg=0.874904 VRAM=39.5GiB | 18.7% done | ETA(epoch): 1085s + [Epoch 56][20/66] loss=0.860693 avg=0.849119 VRAM=39.2GiB | 18.8% done | ETA(epoch): 892s + [Epoch 56][30/66] loss=0.784403 avg=0.844763 VRAM=39.5GiB | 18.8% done | ETA(epoch): 699s + [Epoch 56][40/66] loss=0.853831 avg=0.849152 VRAM=39.2GiB | 18.9% done | ETA(epoch): 504s + [Epoch 56][50/66] loss=0.912882 avg=0.845173 VRAM=39.5GiB | 18.9% done | ETA(epoch): 310s + [Epoch 56][60/66] loss=0.824275 avg=0.838263 VRAM=39.2GiB | 19.0% done | ETA(epoch): 116s + Train loss: 0.838852 (1276.3s) ETA: 5709min + Val loss: 0.854097 [t_0.0-0.2=1.0823 t_0.2-0.4=0.9704 t_0.4-0.6=0.8426 t_0.6-0.8=0.7198 t_0.8-1.0=0.6678] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 57/299 (19% done) --- + [Epoch 57][10/66] loss=0.899918 avg=0.850229 VRAM=39.5GiB | 19.1% done | ETA(epoch): 1083s + [Epoch 57][20/66] loss=0.894635 avg=0.842480 VRAM=39.2GiB | 19.1% done | ETA(epoch): 890s + [Epoch 57][30/66] loss=0.786576 avg=0.845797 VRAM=39.5GiB | 19.2% done | ETA(epoch): 696s + [Epoch 57][40/66] loss=0.813776 avg=0.852517 VRAM=39.2GiB | 19.2% done | ETA(epoch): 502s + [Epoch 57][50/66] loss=0.842313 avg=0.855900 VRAM=39.5GiB | 19.3% done | ETA(epoch): 309s + [Epoch 57][60/66] loss=0.963216 avg=0.864021 VRAM=39.2GiB | 19.3% done | ETA(epoch): 116s + Train loss: 0.863094 (1274.8s) ETA: 5683min + Val loss: 0.838256 [t_0.0-0.2=1.0919 t_0.2-0.4=1.0062 t_0.4-0.6=0.8283 t_0.6-0.8=0.7130 t_0.8-1.0=0.6689] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 58/299 (19% done) --- + [Epoch 58][10/66] loss=0.738860 avg=0.826805 VRAM=39.5GiB | 19.4% done | ETA(epoch): 1087s + [Epoch 58][20/66] loss=0.822210 avg=0.824538 VRAM=39.2GiB | 19.4% done | ETA(epoch): 895s + [Epoch 58][30/66] loss=0.836969 avg=0.834181 VRAM=39.5GiB | 19.5% done | ETA(epoch): 698s + [Epoch 58][40/66] loss=0.810203 avg=0.834758 VRAM=39.2GiB | 19.5% done | ETA(epoch): 504s + [Epoch 58][50/66] loss=0.858481 avg=0.845276 VRAM=39.5GiB | 19.6% done | ETA(epoch): 310s + [Epoch 58][60/66] loss=0.884294 avg=0.851421 VRAM=39.2GiB | 19.6% done | ETA(epoch): 116s + Train loss: 0.856940 (1277.1s) ETA: 5657min + Val loss: 0.863984 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9502 t_0.4-0.6=0.8556 t_0.6-0.8=0.7172 t_0.8-1.0=0.6643] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 59/299 (20% done) --- + [Epoch 59][10/66] loss=0.907514 avg=0.865797 VRAM=39.5GiB | 19.7% done | ETA(epoch): 1084s + [Epoch 59][20/66] loss=0.983960 avg=0.888302 VRAM=39.2GiB | 19.8% done | ETA(epoch): 889s + [Epoch 59][30/66] loss=0.854637 avg=0.875648 VRAM=39.5GiB | 19.8% done | ETA(epoch): 695s + [Epoch 59][40/66] loss=0.866375 avg=0.868926 VRAM=39.2GiB | 19.9% done | ETA(epoch): 502s + [Epoch 59][50/66] loss=0.926141 avg=0.860852 VRAM=39.5GiB | 19.9% done | ETA(epoch): 309s + [Epoch 59][60/66] loss=0.890899 avg=0.864442 VRAM=39.2GiB | 20.0% done | ETA(epoch): 116s + Train loss: 0.868420 (1279.3s) ETA: 5631min + Val loss: 0.854080 [t_0.0-0.2=1.0600 t_0.2-0.4=0.9737 t_0.4-0.6=0.8753 t_0.6-0.8=0.7340 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 60/299 (20% done) --- + [MilestoneVis] train_0 step 3960 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3960 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3960 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3960 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3960 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3960 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3960 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3960 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3960 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3960 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + [Epoch 60][10/66] loss=0.785737 avg=0.855034 VRAM=39.5GiB | 20.1% done | ETA(epoch): 6824s + [Epoch 60][20/66] loss=0.843651 avg=0.887663 VRAM=39.2GiB | 20.1% done | ETA(epoch): 3251s + [Epoch 60][30/66] loss=0.870512 avg=0.879788 VRAM=39.5GiB | 20.2% done | ETA(epoch): 1929s + [Epoch 60][40/66] loss=0.774783 avg=0.870265 VRAM=39.2GiB | 20.2% done | ETA(epoch): 1171s + [Epoch 60][50/66] loss=0.861740 avg=0.865483 VRAM=39.5GiB | 20.3% done | ETA(epoch): 638s + [Epoch 60][60/66] loss=0.858366 avg=0.867000 VRAM=39.2GiB | 20.3% done | ETA(epoch): 219s + Train loss: 0.865430 (2309.3s) ETA: 5673min + Val loss: 0.880664 [t_0.0-0.2=1.0885 t_0.2-0.4=1.0069 t_0.4-0.6=0.8595 t_0.6-0.8=0.7233 t_0.8-1.0=0.6586] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 61/299 (20% done) --- + [Epoch 61][10/66] loss=0.828844 avg=0.839243 VRAM=39.5GiB | 20.4% done | ETA(epoch): 1084s + [Epoch 61][20/66] loss=0.791276 avg=0.864663 VRAM=39.2GiB | 20.4% done | ETA(epoch): 889s + [Epoch 61][30/66] loss=0.804754 avg=0.863166 VRAM=39.5GiB | 20.5% done | ETA(epoch): 695s + [Epoch 61][40/66] loss=0.927005 avg=0.870742 VRAM=39.2GiB | 20.5% done | ETA(epoch): 502s + [Epoch 61][50/66] loss=0.758769 avg=0.862447 VRAM=39.5GiB | 20.6% done | ETA(epoch): 309s + [Epoch 61][60/66] loss=0.884402 avg=0.861586 VRAM=39.2GiB | 20.6% done | ETA(epoch): 116s + Train loss: 0.863277 (1273.7s) ETA: 5646min + Val loss: 0.871896 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0054 t_0.4-0.6=0.8011 t_0.6-0.8=0.7211 t_0.8-1.0=0.6599] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 62/299 (21% done) --- + [Epoch 62][10/66] loss=0.903851 avg=0.840744 VRAM=39.5GiB | 20.7% done | ETA(epoch): 1082s + [Epoch 62][20/66] loss=0.826332 avg=0.844696 VRAM=39.2GiB | 20.8% done | ETA(epoch): 888s + [Epoch 62][30/66] loss=0.781579 avg=0.852695 VRAM=39.5GiB | 20.8% done | ETA(epoch): 695s + [Epoch 62][40/66] loss=0.838229 avg=0.848752 VRAM=39.2GiB | 20.9% done | ETA(epoch): 502s + [Epoch 62][50/66] loss=0.978534 avg=0.857705 VRAM=39.5GiB | 20.9% done | ETA(epoch): 310s + [Epoch 62][60/66] loss=0.994611 avg=0.863000 VRAM=39.2GiB | 21.0% done | ETA(epoch): 116s + Train loss: 0.863898 (1278.4s) ETA: 5619min + Val loss: 0.873113 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0056 t_0.4-0.6=0.8289 t_0.6-0.8=0.7106 t_0.8-1.0=0.6640] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 63/299 (21% done) --- + [Epoch 63][10/66] loss=0.870761 avg=0.871764 VRAM=39.5GiB | 21.1% done | ETA(epoch): 1079s + [Epoch 63][20/66] loss=0.818406 avg=0.876434 VRAM=39.2GiB | 21.1% done | ETA(epoch): 886s + [Epoch 63][30/66] loss=0.901306 avg=0.886790 VRAM=39.5GiB | 21.2% done | ETA(epoch): 694s + [Epoch 63][40/66] loss=0.810936 avg=0.877544 VRAM=39.2GiB | 21.2% done | ETA(epoch): 503s + [Epoch 63][50/66] loss=0.842867 avg=0.884534 VRAM=39.5GiB | 21.3% done | ETA(epoch): 310s + [Epoch 63][60/66] loss=0.885507 avg=0.888478 VRAM=39.2GiB | 21.3% done | ETA(epoch): 116s + Train loss: 0.887370 (1278.0s) ETA: 5592min + Val loss: 0.822170 [t_0.0-0.2=1.0733 t_0.2-0.4=0.9645 t_0.4-0.6=0.8273 t_0.6-0.8=0.7339 t_0.8-1.0=0.6393] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 64/299 (21% done) --- + [Epoch 64][10/66] loss=0.809402 avg=0.859239 VRAM=39.5GiB | 21.4% done | ETA(epoch): 1084s + [Epoch 64][20/66] loss=0.982290 avg=0.862494 VRAM=39.2GiB | 21.4% done | ETA(epoch): 889s + [Epoch 64][30/66] loss=0.932577 avg=0.862322 VRAM=39.5GiB | 21.5% done | ETA(epoch): 698s + [Epoch 64][40/66] loss=0.961400 avg=0.857815 VRAM=39.2GiB | 21.5% done | ETA(epoch): 504s + [Epoch 64][50/66] loss=0.995040 avg=0.861931 VRAM=39.5GiB | 21.6% done | ETA(epoch): 310s + [Epoch 64][60/66] loss=0.808942 avg=0.856489 VRAM=39.2GiB | 21.6% done | ETA(epoch): 116s + Train loss: 0.859663 (1278.5s) ETA: 5566min + Val loss: 0.883039 [t_0.0-0.2=1.0742 t_0.2-0.4=0.9749 t_0.4-0.6=0.8609 t_0.6-0.8=0.7362 t_0.8-1.0=0.6688] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 65/299 (22% done) --- + [Epoch 65][10/66] loss=0.820082 avg=0.871409 VRAM=39.5GiB | 21.7% done | ETA(epoch): 1086s + [Epoch 65][20/66] loss=0.907504 avg=0.871621 VRAM=39.2GiB | 21.8% done | ETA(epoch): 893s + [Epoch 65][30/66] loss=0.930645 avg=0.875813 VRAM=39.5GiB | 21.8% done | ETA(epoch): 699s + [Epoch 65][40/66] loss=0.896161 avg=0.869930 VRAM=39.2GiB | 21.9% done | ETA(epoch): 505s + [Epoch 65][50/66] loss=0.692762 avg=0.866270 VRAM=39.5GiB | 21.9% done | ETA(epoch): 311s + [Epoch 65][60/66] loss=0.869640 avg=0.865736 VRAM=39.2GiB | 22.0% done | ETA(epoch): 116s + Train loss: 0.868992 (1279.5s) ETA: 5539min + Val loss: 0.846680 [t_0.0-0.2=1.0753 t_0.2-0.4=1.0033 t_0.4-0.6=0.8387 t_0.6-0.8=0.7204 t_0.8-1.0=0.6712] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 66/299 (22% done) --- + [Epoch 66][10/66] loss=0.757571 avg=0.864408 VRAM=39.5GiB | 22.1% done | ETA(epoch): 1085s + [Epoch 66][20/66] loss=0.816697 avg=0.859915 VRAM=39.2GiB | 22.1% done | ETA(epoch): 890s + [Epoch 66][30/66] loss=0.801672 avg=0.866122 VRAM=39.5GiB | 22.2% done | ETA(epoch): 696s + [Epoch 66][40/66] loss=0.717501 avg=0.863648 VRAM=39.2GiB | 22.2% done | ETA(epoch): 502s + [Epoch 66][50/66] loss=0.912715 avg=0.859251 VRAM=39.5GiB | 22.3% done | ETA(epoch): 309s + [Epoch 66][60/66] loss=0.936876 avg=0.854087 VRAM=39.2GiB | 22.3% done | ETA(epoch): 116s + Train loss: 0.846303 (1274.4s) ETA: 5513min + Val loss: 0.844110 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9982 t_0.4-0.6=0.8402 t_0.6-0.8=0.7006 t_0.8-1.0=0.6459] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 67/299 (22% done) --- + [Epoch 67][10/66] loss=0.802491 avg=0.850050 VRAM=39.5GiB | 22.4% done | ETA(epoch): 1084s + [Epoch 67][20/66] loss=0.881818 avg=0.854556 VRAM=39.2GiB | 22.4% done | ETA(epoch): 889s + [Epoch 67][30/66] loss=0.916557 avg=0.856152 VRAM=39.5GiB | 22.5% done | ETA(epoch): 697s + [Epoch 67][40/66] loss=0.839413 avg=0.858907 VRAM=39.2GiB | 22.5% done | ETA(epoch): 503s + [Epoch 67][50/66] loss=0.815991 avg=0.863124 VRAM=39.5GiB | 22.6% done | ETA(epoch): 309s + [Epoch 67][60/66] loss=0.901874 avg=0.860269 VRAM=39.2GiB | 22.6% done | ETA(epoch): 116s + Train loss: 0.857495 (1275.1s) ETA: 5486min + Val loss: 0.882120 [t_0.0-0.2=1.0812 t_0.2-0.4=1.0044 t_0.4-0.6=0.8309 t_0.6-0.8=0.7479 t_0.8-1.0=0.6364] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 68/299 (23% done) --- + [Epoch 68][10/66] loss=0.848072 avg=0.861213 VRAM=39.5GiB | 22.7% done | ETA(epoch): 1086s + [Epoch 68][20/66] loss=0.966660 avg=0.870547 VRAM=39.2GiB | 22.8% done | ETA(epoch): 894s + [Epoch 68][30/66] loss=0.879253 avg=0.863410 VRAM=39.5GiB | 22.8% done | ETA(epoch): 700s + [Epoch 68][40/66] loss=0.960837 avg=0.856823 VRAM=39.2GiB | 22.9% done | ETA(epoch): 505s + [Epoch 68][50/66] loss=0.979323 avg=0.860386 VRAM=39.5GiB | 22.9% done | ETA(epoch): 311s + [Epoch 68][60/66] loss=0.859260 avg=0.865319 VRAM=39.2GiB | 23.0% done | ETA(epoch): 117s + Train loss: 0.867092 (1284.6s) ETA: 5461min + Val loss: 0.838484 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9658 t_0.4-0.6=0.8321 t_0.6-0.8=0.7162 t_0.8-1.0=0.6776] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 69/299 (23% done) --- + [Epoch 69][10/66] loss=0.886718 avg=0.880164 VRAM=39.5GiB | 23.1% done | ETA(epoch): 1089s + [Epoch 69][20/66] loss=0.838590 avg=0.860817 VRAM=39.2GiB | 23.1% done | ETA(epoch): 893s + [Epoch 69][30/66] loss=0.843403 avg=0.861292 VRAM=39.5GiB | 23.2% done | ETA(epoch): 697s + [Epoch 69][40/66] loss=0.861732 avg=0.855981 VRAM=39.2GiB | 23.2% done | ETA(epoch): 503s + [Epoch 69][50/66] loss=0.887530 avg=0.848529 VRAM=39.5GiB | 23.3% done | ETA(epoch): 309s + [Epoch 69][60/66] loss=0.729708 avg=0.844824 VRAM=39.2GiB | 23.3% done | ETA(epoch): 116s + Train loss: 0.842883 (1274.8s) ETA: 5434min + Val loss: 0.866195 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9917 t_0.4-0.6=0.8601 t_0.6-0.8=0.7312 t_0.8-1.0=0.6542] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 70/299 (23% done) --- + [Epoch 70][10/66] loss=0.801966 avg=0.877653 VRAM=39.5GiB | 23.4% done | ETA(epoch): 1083s + [Epoch 70][20/66] loss=0.876961 avg=0.848236 VRAM=39.2GiB | 23.4% done | ETA(epoch): 888s + [Epoch 70][30/66] loss=0.909231 avg=0.852374 VRAM=39.5GiB | 23.5% done | ETA(epoch): 695s + [Epoch 70][40/66] loss=0.898438 avg=0.865837 VRAM=39.2GiB | 23.5% done | ETA(epoch): 502s + [Epoch 70][50/66] loss=0.817959 avg=0.864854 VRAM=39.5GiB | 23.6% done | ETA(epoch): 309s + [Epoch 70][60/66] loss=0.836258 avg=0.857262 VRAM=39.2GiB | 23.6% done | ETA(epoch): 116s + Train loss: 0.856827 (1272.9s) ETA: 5408min + Val loss: 0.823099 [t_0.0-0.2=1.0742 t_0.2-0.4=0.9741 t_0.4-0.6=0.8179 t_0.6-0.8=0.7257 t_0.8-1.0=0.6449] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 71/299 (24% done) --- + [Epoch 71][10/66] loss=0.787999 avg=0.847721 VRAM=39.5GiB | 23.7% done | ETA(epoch): 1084s + [Epoch 71][20/66] loss=0.810563 avg=0.848950 VRAM=39.2GiB | 23.8% done | ETA(epoch): 892s + [Epoch 71][30/66] loss=0.863622 avg=0.850743 VRAM=39.5GiB | 23.8% done | ETA(epoch): 698s + [Epoch 71][40/66] loss=0.724628 avg=0.846229 VRAM=39.2GiB | 23.9% done | ETA(epoch): 504s + [Epoch 71][50/66] loss=0.858565 avg=0.851272 VRAM=39.5GiB | 23.9% done | ETA(epoch): 310s + [Epoch 71][60/66] loss=0.911413 avg=0.849969 VRAM=39.2GiB | 24.0% done | ETA(epoch): 116s + Train loss: 0.851919 (1277.6s) ETA: 5382min + Val loss: 0.858669 [t_0.0-0.2=1.0873 t_0.2-0.4=0.9824 t_0.4-0.6=0.8382 t_0.6-0.8=0.7136 t_0.8-1.0=0.6453] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 72/299 (24% done) --- + [Epoch 72][10/66] loss=0.919283 avg=0.825420 VRAM=39.5GiB | 24.1% done | ETA(epoch): 1081s + [Epoch 72][20/66] loss=0.779112 avg=0.836696 VRAM=39.2GiB | 24.1% done | ETA(epoch): 887s + [Epoch 72][30/66] loss=0.870403 avg=0.847604 VRAM=39.5GiB | 24.2% done | ETA(epoch): 694s + [Epoch 72][40/66] loss=0.828432 avg=0.855235 VRAM=39.2GiB | 24.2% done | ETA(epoch): 501s + [Epoch 72][50/66] loss=0.858346 avg=0.849646 VRAM=39.5GiB | 24.3% done | ETA(epoch): 308s + [Epoch 72][60/66] loss=0.940843 avg=0.847286 VRAM=39.2GiB | 24.3% done | ETA(epoch): 116s + Train loss: 0.847286 (1270.9s) ETA: 5356min + Val loss: 0.877648 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9934 t_0.4-0.6=0.8637 t_0.6-0.8=0.6929 t_0.8-1.0=0.6551] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 73/299 (24% done) --- + [Epoch 73][10/66] loss=0.784809 avg=0.854907 VRAM=39.5GiB | 24.4% done | ETA(epoch): 1081s + [Epoch 73][20/66] loss=0.796991 avg=0.875843 VRAM=39.2GiB | 24.4% done | ETA(epoch): 888s + [Epoch 73][30/66] loss=1.023444 avg=0.865494 VRAM=39.5GiB | 24.5% done | ETA(epoch): 696s + [Epoch 73][40/66] loss=0.939650 avg=0.847901 VRAM=39.2GiB | 24.5% done | ETA(epoch): 502s + [Epoch 73][50/66] loss=0.912026 avg=0.851265 VRAM=39.5GiB | 24.6% done | ETA(epoch): 309s + [Epoch 73][60/66] loss=0.961393 avg=0.858858 VRAM=39.2GiB | 24.6% done | ETA(epoch): 116s + Train loss: 0.858867 (1272.7s) ETA: 5330min + Val loss: 0.848048 [t_0.0-0.2=1.0561 t_0.2-0.4=0.9663 t_0.4-0.6=0.8734 t_0.6-0.8=0.6995 t_0.8-1.0=0.6609] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 74/299 (25% done) --- + [Epoch 74][10/66] loss=0.887224 avg=0.819717 VRAM=39.5GiB | 24.7% done | ETA(epoch): 1103s + [Epoch 74][20/66] loss=0.891122 avg=0.862795 VRAM=39.2GiB | 24.8% done | ETA(epoch): 897s + [Epoch 74][30/66] loss=0.769519 avg=0.852439 VRAM=39.5GiB | 24.8% done | ETA(epoch): 701s + [Epoch 74][40/66] loss=0.963690 avg=0.852176 VRAM=39.2GiB | 24.9% done | ETA(epoch): 506s + [Epoch 74][50/66] loss=0.871627 avg=0.856266 VRAM=39.5GiB | 24.9% done | ETA(epoch): 311s + [Epoch 74][60/66] loss=0.790252 avg=0.853891 VRAM=39.2GiB | 25.0% done | ETA(epoch): 116s + Train loss: 0.851401 (1278.8s) ETA: 5304min + Val loss: 0.863438 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0246 t_0.4-0.6=0.8386 t_0.6-0.8=0.7216 t_0.8-1.0=0.6598] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 75/299 (25% done) --- + [Epoch 75][10/66] loss=0.808432 avg=0.869825 VRAM=39.5GiB | 25.1% done | ETA(epoch): 1081s + [Epoch 75][20/66] loss=0.814575 avg=0.842033 VRAM=39.2GiB | 25.1% done | ETA(epoch): 887s + [Epoch 75][30/66] loss=0.953917 avg=0.858870 VRAM=39.5GiB | 25.2% done | ETA(epoch): 694s + [Epoch 75][40/66] loss=0.823400 avg=0.852926 VRAM=39.2GiB | 25.2% done | ETA(epoch): 501s + [Epoch 75][50/66] loss=0.765319 avg=0.855212 VRAM=39.5GiB | 25.3% done | ETA(epoch): 308s + [Epoch 75][60/66] loss=0.968925 avg=0.858823 VRAM=39.2GiB | 25.3% done | ETA(epoch): 116s + Train loss: 0.860490 (1272.2s) ETA: 5278min + Val loss: 0.834139 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9853 t_0.4-0.6=0.8365 t_0.6-0.8=0.7223 t_0.8-1.0=0.6541] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 76/299 (25% done) --- + [Epoch 76][10/66] loss=0.857084 avg=0.855610 VRAM=39.5GiB | 25.4% done | ETA(epoch): 1084s + [Epoch 76][20/66] loss=0.843200 avg=0.851478 VRAM=39.2GiB | 25.4% done | ETA(epoch): 891s + [Epoch 76][30/66] loss=0.754067 avg=0.854431 VRAM=39.5GiB | 25.5% done | ETA(epoch): 696s + [Epoch 76][40/66] loss=0.832825 avg=0.853983 VRAM=39.2GiB | 25.5% done | ETA(epoch): 502s + [Epoch 76][50/66] loss=0.930456 avg=0.854776 VRAM=39.5GiB | 25.6% done | ETA(epoch): 309s + [Epoch 76][60/66] loss=0.971158 avg=0.854973 VRAM=39.2GiB | 25.6% done | ETA(epoch): 116s + Train loss: 0.857077 (1276.7s) ETA: 5253min + Val loss: 0.860647 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0306 t_0.4-0.6=0.8185 t_0.6-0.8=0.7348 t_0.8-1.0=0.6577] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 77/299 (26% done) --- + [Epoch 77][10/66] loss=0.787339 avg=0.846020 VRAM=39.5GiB | 25.7% done | ETA(epoch): 1085s + [Epoch 77][20/66] loss=0.923913 avg=0.853486 VRAM=39.2GiB | 25.8% done | ETA(epoch): 893s + [Epoch 77][30/66] loss=0.897173 avg=0.866559 VRAM=39.5GiB | 25.8% done | ETA(epoch): 699s + [Epoch 77][40/66] loss=0.750171 avg=0.864188 VRAM=39.2GiB | 25.9% done | ETA(epoch): 504s + [Epoch 77][50/66] loss=0.875897 avg=0.865339 VRAM=39.5GiB | 25.9% done | ETA(epoch): 310s + [Epoch 77][60/66] loss=0.774228 avg=0.865548 VRAM=39.2GiB | 26.0% done | ETA(epoch): 116s + Train loss: 0.865013 (1275.6s) ETA: 5227min + Val loss: 0.837265 [t_0.0-0.2=1.0605 t_0.2-0.4=0.9900 t_0.4-0.6=0.8420 t_0.6-0.8=0.7311 t_0.8-1.0=0.6483] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 78/299 (26% done) --- + [Epoch 78][10/66] loss=0.737374 avg=0.816285 VRAM=39.5GiB | 26.1% done | ETA(epoch): 1085s + [Epoch 78][20/66] loss=0.734539 avg=0.836585 VRAM=39.2GiB | 26.1% done | ETA(epoch): 893s + [Epoch 78][30/66] loss=0.887889 avg=0.853968 VRAM=39.5GiB | 26.2% done | ETA(epoch): 699s + [Epoch 78][40/66] loss=0.741871 avg=0.851977 VRAM=39.2GiB | 26.2% done | ETA(epoch): 505s + [Epoch 78][50/66] loss=0.794363 avg=0.851222 VRAM=39.5GiB | 26.3% done | ETA(epoch): 310s + [Epoch 78][60/66] loss=0.811224 avg=0.855714 VRAM=39.2GiB | 26.3% done | ETA(epoch): 116s + Train loss: 0.861196 (1277.5s) ETA: 5202min + Val loss: 0.818382 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9952 t_0.4-0.6=0.7945 t_0.6-0.8=0.7264 t_0.8-1.0=0.6390] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0078 (BEST) + Deleted old checkpoint: checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 79/299 (26% done) --- + [Epoch 79][10/66] loss=0.968794 avg=0.878037 VRAM=39.5GiB | 26.4% done | ETA(epoch): 1080s + [Epoch 79][20/66] loss=0.804184 avg=0.879424 VRAM=39.2GiB | 26.4% done | ETA(epoch): 886s + [Epoch 79][30/66] loss=0.899224 avg=0.865221 VRAM=39.5GiB | 26.5% done | ETA(epoch): 694s + [Epoch 79][40/66] loss=0.815168 avg=0.857301 VRAM=39.2GiB | 26.5% done | ETA(epoch): 502s + [Epoch 79][50/66] loss=0.912328 avg=0.867541 VRAM=39.5GiB | 26.6% done | ETA(epoch): 309s + [Epoch 79][60/66] loss=0.785146 avg=0.866631 VRAM=39.2GiB | 26.6% done | ETA(epoch): 116s + Train loss: 0.860410 (1272.5s) ETA: 5176min + Val loss: 0.858606 [t_0.0-0.2=1.0790 t_0.2-0.4=0.9641 t_0.4-0.6=0.8705 t_0.6-0.8=0.7508 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 80/299 (27% done) --- + [Epoch 80][10/66] loss=0.958360 avg=0.873184 VRAM=39.5GiB | 26.7% done | ETA(epoch): 1082s + [Epoch 80][20/66] loss=0.956547 avg=0.865148 VRAM=39.2GiB | 26.8% done | ETA(epoch): 887s + [Epoch 80][30/66] loss=0.839649 avg=0.852472 VRAM=39.5GiB | 26.8% done | ETA(epoch): 694s + [Epoch 80][40/66] loss=0.921797 avg=0.853147 VRAM=39.2GiB | 26.9% done | ETA(epoch): 502s + [Epoch 80][50/66] loss=0.805308 avg=0.852902 VRAM=39.5GiB | 26.9% done | ETA(epoch): 309s + [Epoch 80][60/66] loss=1.026982 avg=0.858626 VRAM=39.2GiB | 27.0% done | ETA(epoch): 116s + Train loss: 0.858052 (1272.1s) ETA: 5151min + Val loss: 0.864453 [t_0.0-0.2=1.0616 t_0.2-0.4=0.9706 t_0.4-0.6=0.8685 t_0.6-0.8=0.7139 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 81/299 (27% done) --- + [Epoch 81][10/66] loss=0.793687 avg=0.848495 VRAM=39.5GiB | 27.1% done | ETA(epoch): 1085s + [Epoch 81][20/66] loss=0.849439 avg=0.843153 VRAM=39.2GiB | 27.1% done | ETA(epoch): 893s + [Epoch 81][30/66] loss=0.911410 avg=0.840885 VRAM=39.5GiB | 27.2% done | ETA(epoch): 699s + [Epoch 81][40/66] loss=0.784433 avg=0.847369 VRAM=39.2GiB | 27.2% done | ETA(epoch): 505s + [Epoch 81][50/66] loss=0.828490 avg=0.846125 VRAM=39.5GiB | 27.3% done | ETA(epoch): 311s + [Epoch 81][60/66] loss=0.903731 avg=0.842440 VRAM=39.2GiB | 27.3% done | ETA(epoch): 117s + Train loss: 0.844832 (1281.3s) ETA: 5126min + Val loss: 0.886003 [t_0.0-0.2=1.0768 t_0.2-0.4=0.9894 t_0.4-0.6=0.8226 t_0.6-0.8=0.7191 t_0.8-1.0=0.6279] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0081 +[MEM @ epoch 81 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 82/299 (27% done) --- + [Epoch 82][10/66] loss=0.765895 avg=0.871694 VRAM=39.5GiB | 27.4% done | ETA(epoch): 1080s + [Epoch 82][20/66] loss=0.795424 avg=0.847827 VRAM=39.2GiB | 27.4% done | ETA(epoch): 886s + [Epoch 82][30/66] loss=0.845010 avg=0.850925 VRAM=39.5GiB | 27.5% done | ETA(epoch): 694s + [Epoch 82][40/66] loss=0.861596 avg=0.851546 VRAM=39.2GiB | 27.5% done | ETA(epoch): 501s + [Epoch 82][50/66] loss=0.838493 avg=0.852002 VRAM=39.5GiB | 27.6% done | ETA(epoch): 308s + [Epoch 82][60/66] loss=0.692428 avg=0.845354 VRAM=39.2GiB | 27.6% done | ETA(epoch): 116s + Train loss: 0.848680 (1271.4s) ETA: 5100min + Val loss: 0.894235 [t_0.0-0.2=1.0734 t_0.2-0.4=1.0031 t_0.4-0.6=0.8262 t_0.6-0.8=0.7204 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 83/299 (28% done) --- + [Epoch 83][10/66] loss=0.744547 avg=0.857464 VRAM=39.5GiB | 27.7% done | ETA(epoch): 1084s + [Epoch 83][20/66] loss=0.745913 avg=0.838835 VRAM=39.2GiB | 27.8% done | ETA(epoch): 889s + [Epoch 83][30/66] loss=0.887356 avg=0.844789 VRAM=39.5GiB | 27.8% done | ETA(epoch): 695s + [Epoch 83][40/66] loss=0.738964 avg=0.847568 VRAM=39.2GiB | 27.9% done | ETA(epoch): 501s + [Epoch 83][50/66] loss=0.881693 avg=0.848453 VRAM=39.5GiB | 27.9% done | ETA(epoch): 308s + [Epoch 83][60/66] loss=0.891287 avg=0.851836 VRAM=39.2GiB | 28.0% done | ETA(epoch): 116s + Train loss: 0.853613 (1272.3s) ETA: 5075min + Val loss: 0.864308 [t_0.0-0.2=1.0701 t_0.2-0.4=1.0395 t_0.4-0.6=0.7981 t_0.6-0.8=0.7215 t_0.8-1.0=0.6611] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 84/299 (28% done) --- + [Epoch 84][10/66] loss=0.796775 avg=0.832319 VRAM=39.5GiB | 28.1% done | ETA(epoch): 1085s + [Epoch 84][20/66] loss=0.732307 avg=0.837062 VRAM=39.2GiB | 28.1% done | ETA(epoch): 891s + [Epoch 84][30/66] loss=0.910845 avg=0.844924 VRAM=39.5GiB | 28.2% done | ETA(epoch): 696s + [Epoch 84][40/66] loss=0.725317 avg=0.846190 VRAM=39.2GiB | 28.2% done | ETA(epoch): 502s + [Epoch 84][50/66] loss=0.881411 avg=0.845714 VRAM=39.5GiB | 28.3% done | ETA(epoch): 310s + [Epoch 84][60/66] loss=0.698267 avg=0.843571 VRAM=39.2GiB | 28.3% done | ETA(epoch): 116s + Train loss: 0.844590 (1279.1s) ETA: 5050min + Val loss: 0.846823 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9994 t_0.4-0.6=0.8570 t_0.6-0.8=0.7066 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 85/299 (28% done) --- + [Epoch 85][10/66] loss=0.907670 avg=0.853793 VRAM=39.5GiB | 28.4% done | ETA(epoch): 1086s + [Epoch 85][20/66] loss=0.860445 avg=0.839784 VRAM=39.2GiB | 28.4% done | ETA(epoch): 893s + [Epoch 85][30/66] loss=0.858063 avg=0.851090 VRAM=39.5GiB | 28.5% done | ETA(epoch): 699s + [Epoch 85][40/66] loss=0.762558 avg=0.852602 VRAM=39.2GiB | 28.5% done | ETA(epoch): 507s + [Epoch 85][50/66] loss=0.816867 avg=0.852891 VRAM=39.5GiB | 28.6% done | ETA(epoch): 312s + [Epoch 85][60/66] loss=0.892475 avg=0.851134 VRAM=39.2GiB | 28.6% done | ETA(epoch): 117s + Train loss: 0.857413 (1284.7s) ETA: 5025min + Val loss: 0.861577 [t_0.0-0.2=1.0724 t_0.2-0.4=1.0008 t_0.4-0.6=0.8655 t_0.6-0.8=0.7119 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 86/299 (29% done) --- + [Epoch 86][10/66] loss=0.983323 avg=0.876445 VRAM=39.5GiB | 28.7% done | ETA(epoch): 1086s + [Epoch 86][20/66] loss=0.833417 avg=0.877650 VRAM=39.2GiB | 28.8% done | ETA(epoch): 894s + [Epoch 86][30/66] loss=0.875804 avg=0.865092 VRAM=39.5GiB | 28.8% done | ETA(epoch): 699s + [Epoch 86][40/66] loss=0.827371 avg=0.855938 VRAM=39.2GiB | 28.9% done | ETA(epoch): 505s + [Epoch 86][50/66] loss=0.855602 avg=0.856537 VRAM=39.5GiB | 28.9% done | ETA(epoch): 311s + [Epoch 86][60/66] loss=0.972358 avg=0.858628 VRAM=39.2GiB | 29.0% done | ETA(epoch): 116s + Train loss: 0.857398 (1278.9s) ETA: 5000min + Val loss: 0.823985 [t_0.0-0.2=1.0818 t_0.2-0.4=0.9753 t_0.4-0.6=0.8720 t_0.6-0.8=0.7172 t_0.8-1.0=0.6440] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 87/299 (29% done) --- + [Epoch 87][10/66] loss=0.958098 avg=0.836855 VRAM=39.5GiB | 29.1% done | ETA(epoch): 1084s + [Epoch 87][20/66] loss=0.788285 avg=0.846516 VRAM=39.2GiB | 29.1% done | ETA(epoch): 892s + [Epoch 87][30/66] loss=0.808937 avg=0.843117 VRAM=39.5GiB | 29.2% done | ETA(epoch): 699s + [Epoch 87][40/66] loss=0.804292 avg=0.839463 VRAM=39.2GiB | 29.2% done | ETA(epoch): 505s + [Epoch 87][50/66] loss=0.911288 avg=0.851481 VRAM=39.5GiB | 29.3% done | ETA(epoch): 311s + [Epoch 87][60/66] loss=0.869663 avg=0.853900 VRAM=39.2GiB | 29.3% done | ETA(epoch): 116s + Train loss: 0.854423 (1280.3s) ETA: 4976min + Val loss: 0.877383 [t_0.0-0.2=1.0675 t_0.2-0.4=0.9741 t_0.4-0.6=0.8148 t_0.6-0.8=0.7174 t_0.8-1.0=0.6390] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 88/299 (29% done) --- + [Epoch 88][10/66] loss=0.810992 avg=0.892482 VRAM=39.5GiB | 29.4% done | ETA(epoch): 1096s + [Epoch 88][20/66] loss=0.772494 avg=0.864561 VRAM=39.2GiB | 29.4% done | ETA(epoch): 897s + [Epoch 88][30/66] loss=0.957588 avg=0.859422 VRAM=39.5GiB | 29.5% done | ETA(epoch): 700s + [Epoch 88][40/66] loss=0.794419 avg=0.869872 VRAM=39.2GiB | 29.5% done | ETA(epoch): 505s + [Epoch 88][50/66] loss=0.735726 avg=0.862053 VRAM=39.5GiB | 29.6% done | ETA(epoch): 310s + [Epoch 88][60/66] loss=0.877242 avg=0.864428 VRAM=39.2GiB | 29.6% done | ETA(epoch): 116s + Train loss: 0.857912 (1277.2s) ETA: 4951min + Val loss: 0.896102 [t_0.0-0.2=1.0900 t_0.2-0.4=1.0024 t_0.4-0.6=0.8279 t_0.6-0.8=0.7084 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 89/299 (30% done) --- + [Epoch 89][10/66] loss=0.899280 avg=0.858187 VRAM=39.5GiB | 29.7% done | ETA(epoch): 1085s + [Epoch 89][20/66] loss=0.874763 avg=0.864452 VRAM=39.2GiB | 29.8% done | ETA(epoch): 893s + [Epoch 89][30/66] loss=0.757146 avg=0.869338 VRAM=39.5GiB | 29.8% done | ETA(epoch): 699s + [Epoch 89][40/66] loss=0.880238 avg=0.868401 VRAM=39.2GiB | 29.9% done | ETA(epoch): 505s + [Epoch 89][50/66] loss=0.898082 avg=0.864738 VRAM=39.5GiB | 29.9% done | ETA(epoch): 310s + [Epoch 89][60/66] loss=0.974648 avg=0.869241 VRAM=39.2GiB | 30.0% done | ETA(epoch): 117s + Train loss: 0.864886 (1282.8s) ETA: 4926min + Val loss: 0.853198 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9949 t_0.4-0.6=0.8482 t_0.6-0.8=0.7048 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 90/299 (30% done) --- + [MilestoneVis] train_0 step 5940 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 5940 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 5940 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 5940 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 5940 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 5940 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 5940 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 5940 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 5940 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 5940 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + [Epoch 90][10/66] loss=0.882498 avg=0.869824 VRAM=39.5GiB | 30.1% done | ETA(epoch): 6838s + [Epoch 90][20/66] loss=0.778907 avg=0.858468 VRAM=39.2GiB | 30.1% done | ETA(epoch): 3255s + [Epoch 90][30/66] loss=0.947957 avg=0.854933 VRAM=39.5GiB | 30.2% done | ETA(epoch): 1932s + [Epoch 90][40/66] loss=0.846994 avg=0.858617 VRAM=39.2GiB | 30.2% done | ETA(epoch): 1173s + [Epoch 90][50/66] loss=0.702529 avg=0.860215 VRAM=39.5GiB | 30.3% done | ETA(epoch): 639s + [Epoch 90][60/66] loss=0.904751 avg=0.857782 VRAM=39.2GiB | 30.3% done | ETA(epoch): 219s + Train loss: 0.854057 (2312.1s) ETA: 4941min + Val loss: 0.834677 [t_0.0-0.2=1.0645 t_0.2-0.4=1.0033 t_0.4-0.6=0.8676 t_0.6-0.8=0.7032 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 91/299 (30% done) --- + [Epoch 91][10/66] loss=0.998819 avg=0.839124 VRAM=39.5GiB | 30.4% done | ETA(epoch): 1084s + [Epoch 91][20/66] loss=0.905316 avg=0.829799 VRAM=39.2GiB | 30.4% done | ETA(epoch): 892s + [Epoch 91][30/66] loss=0.766159 avg=0.834183 VRAM=39.5GiB | 30.5% done | ETA(epoch): 699s + [Epoch 91][40/66] loss=0.943132 avg=0.838166 VRAM=39.2GiB | 30.5% done | ETA(epoch): 505s + [Epoch 91][50/66] loss=0.994539 avg=0.844494 VRAM=39.5GiB | 30.6% done | ETA(epoch): 311s + [Epoch 91][60/66] loss=0.741378 avg=0.842725 VRAM=39.2GiB | 30.6% done | ETA(epoch): 116s + Train loss: 0.840263 (1281.0s) ETA: 4916min + Val loss: 0.825243 [t_0.0-0.2=1.0728 t_0.2-0.4=1.0095 t_0.4-0.6=0.8491 t_0.6-0.8=0.7274 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 92/299 (31% done) --- + [Epoch 92][10/66] loss=0.938607 avg=0.871684 VRAM=39.5GiB | 30.7% done | ETA(epoch): 1086s + [Epoch 92][20/66] loss=0.913421 avg=0.861608 VRAM=39.2GiB | 30.8% done | ETA(epoch): 893s + [Epoch 92][30/66] loss=0.874223 avg=0.857266 VRAM=39.5GiB | 30.8% done | ETA(epoch): 699s + [Epoch 92][40/66] loss=0.782172 avg=0.855591 VRAM=39.2GiB | 30.9% done | ETA(epoch): 505s + [Epoch 92][50/66] loss=0.785419 avg=0.858220 VRAM=39.5GiB | 30.9% done | ETA(epoch): 311s + [Epoch 92][60/66] loss=0.794539 avg=0.853197 VRAM=39.2GiB | 31.0% done | ETA(epoch): 116s + Train loss: 0.852780 (1280.0s) ETA: 4890min + Val loss: 0.857920 [t_0.0-0.2=1.0743 t_0.2-0.4=0.9803 t_0.4-0.6=0.8761 t_0.6-0.8=0.7046 t_0.8-1.0=0.6523] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 93/299 (31% done) --- + [Epoch 93][10/66] loss=0.898972 avg=0.869134 VRAM=39.5GiB | 31.1% done | ETA(epoch): 1084s + [Epoch 93][20/66] loss=0.691336 avg=0.853264 VRAM=39.2GiB | 31.1% done | ETA(epoch): 889s + [Epoch 93][30/66] loss=0.800095 avg=0.855500 VRAM=39.5GiB | 31.2% done | ETA(epoch): 695s + [Epoch 93][40/66] loss=0.879268 avg=0.856967 VRAM=39.2GiB | 31.2% done | ETA(epoch): 502s + [Epoch 93][50/66] loss=0.935391 avg=0.856510 VRAM=39.5GiB | 31.3% done | ETA(epoch): 309s + [Epoch 93][60/66] loss=0.804608 avg=0.857832 VRAM=39.2GiB | 31.3% done | ETA(epoch): 116s + Train loss: 0.856543 (1273.1s) ETA: 4865min + Val loss: 0.876832 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9828 t_0.4-0.6=0.8083 t_0.6-0.8=0.7254 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 94/299 (31% done) --- + [Epoch 94][10/66] loss=0.875484 avg=0.829181 VRAM=39.5GiB | 31.4% done | ETA(epoch): 1081s + [Epoch 94][20/66] loss=0.760647 avg=0.841199 VRAM=39.2GiB | 31.4% done | ETA(epoch): 888s + [Epoch 94][30/66] loss=0.822967 avg=0.847461 VRAM=39.5GiB | 31.5% done | ETA(epoch): 699s + [Epoch 94][40/66] loss=0.787375 avg=0.851113 VRAM=39.2GiB | 31.5% done | ETA(epoch): 504s + [Epoch 94][50/66] loss=0.926629 avg=0.851308 VRAM=39.5GiB | 31.6% done | ETA(epoch): 310s + [Epoch 94][60/66] loss=0.815333 avg=0.850487 VRAM=39.2GiB | 31.6% done | ETA(epoch): 116s + Train loss: 0.852989 (1278.4s) ETA: 4840min + Val loss: 0.868274 [t_0.0-0.2=1.0721 t_0.2-0.4=1.0108 t_0.4-0.6=0.8603 t_0.6-0.8=0.7051 t_0.8-1.0=0.6490] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 95/299 (32% done) --- + [Epoch 95][10/66] loss=0.904658 avg=0.853979 VRAM=39.5GiB | 31.7% done | ETA(epoch): 1084s + [Epoch 95][20/66] loss=0.873471 avg=0.864816 VRAM=39.2GiB | 31.8% done | ETA(epoch): 896s + [Epoch 95][30/66] loss=0.834193 avg=0.865986 VRAM=39.5GiB | 31.8% done | ETA(epoch): 699s + [Epoch 95][40/66] loss=0.793060 avg=0.860781 VRAM=39.2GiB | 31.9% done | ETA(epoch): 504s + [Epoch 95][50/66] loss=0.844436 avg=0.861724 VRAM=39.5GiB | 31.9% done | ETA(epoch): 310s + [Epoch 95][60/66] loss=0.903543 avg=0.857991 VRAM=39.2GiB | 32.0% done | ETA(epoch): 116s + Train loss: 0.854721 (1277.7s) ETA: 4815min + Val loss: 0.863101 [t_0.0-0.2=1.0675 t_0.2-0.4=0.9879 t_0.4-0.6=0.8366 t_0.6-0.8=0.7318 t_0.8-1.0=0.6654] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 96/299 (32% done) --- + [Epoch 96][10/66] loss=0.957470 avg=0.881664 VRAM=39.5GiB | 32.1% done | ETA(epoch): 1081s + [Epoch 96][20/66] loss=0.791983 avg=0.854811 VRAM=39.2GiB | 32.1% done | ETA(epoch): 888s + [Epoch 96][30/66] loss=0.810405 avg=0.847229 VRAM=39.5GiB | 32.2% done | ETA(epoch): 694s + [Epoch 96][40/66] loss=0.883208 avg=0.840747 VRAM=39.2GiB | 32.2% done | ETA(epoch): 501s + [Epoch 96][50/66] loss=0.753700 avg=0.843736 VRAM=39.5GiB | 32.3% done | ETA(epoch): 309s + [Epoch 96][60/66] loss=0.810085 avg=0.850810 VRAM=39.2GiB | 32.3% done | ETA(epoch): 116s + Train loss: 0.855005 (1272.6s) ETA: 4790min + Val loss: 0.855313 [t_0.0-0.2=1.0682 t_0.2-0.4=0.9733 t_0.4-0.6=0.8260 t_0.6-0.8=0.7024 t_0.8-1.0=0.6500] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 97/299 (32% done) --- + [Epoch 97][10/66] loss=0.846822 avg=0.858427 VRAM=39.5GiB | 32.4% done | ETA(epoch): 1080s + [Epoch 97][20/66] loss=0.815228 avg=0.854630 VRAM=39.2GiB | 32.4% done | ETA(epoch): 887s + [Epoch 97][30/66] loss=0.786421 avg=0.847978 VRAM=39.5GiB | 32.5% done | ETA(epoch): 694s + [Epoch 97][40/66] loss=0.865791 avg=0.840552 VRAM=39.2GiB | 32.5% done | ETA(epoch): 501s + [Epoch 97][50/66] loss=0.886253 avg=0.844738 VRAM=39.5GiB | 32.6% done | ETA(epoch): 309s + [Epoch 97][60/66] loss=0.831386 avg=0.848036 VRAM=39.2GiB | 32.6% done | ETA(epoch): 116s + Train loss: 0.849614 (1279.1s) ETA: 4765min + Val loss: 0.854577 [t_0.0-0.2=1.0890 t_0.2-0.4=1.0178 t_0.4-0.6=0.8335 t_0.6-0.8=0.6992 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 98/299 (33% done) --- + [Epoch 98][10/66] loss=0.896570 avg=0.825269 VRAM=39.5GiB | 32.7% done | ETA(epoch): 1082s + [Epoch 98][20/66] loss=0.881495 avg=0.843705 VRAM=39.2GiB | 32.8% done | ETA(epoch): 890s + [Epoch 98][30/66] loss=0.859637 avg=0.832838 VRAM=39.5GiB | 32.8% done | ETA(epoch): 696s + [Epoch 98][40/66] loss=0.908907 avg=0.838236 VRAM=39.2GiB | 32.9% done | ETA(epoch): 503s + [Epoch 98][50/66] loss=0.884767 avg=0.836769 VRAM=39.5GiB | 32.9% done | ETA(epoch): 309s + [Epoch 98][60/66] loss=0.800848 avg=0.836132 VRAM=39.2GiB | 33.0% done | ETA(epoch): 116s + Train loss: 0.837820 (1275.6s) ETA: 4739min + Val loss: 0.845194 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9798 t_0.4-0.6=0.8041 t_0.6-0.8=0.7190 t_0.8-1.0=0.6482] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 99/299 (33% done) --- + [Epoch 99][10/66] loss=0.735908 avg=0.841370 VRAM=39.5GiB | 33.1% done | ETA(epoch): 1084s + [Epoch 99][20/66] loss=0.778635 avg=0.830437 VRAM=39.2GiB | 33.1% done | ETA(epoch): 892s + [Epoch 99][30/66] loss=0.993354 avg=0.834395 VRAM=39.5GiB | 33.2% done | ETA(epoch): 697s + [Epoch 99][40/66] loss=0.865038 avg=0.843943 VRAM=39.2GiB | 33.2% done | ETA(epoch): 503s + [Epoch 99][50/66] loss=0.840183 avg=0.842080 VRAM=39.5GiB | 33.3% done | ETA(epoch): 309s + [Epoch 99][60/66] loss=0.788811 avg=0.846599 VRAM=39.2GiB | 33.3% done | ETA(epoch): 116s + Train loss: 0.852637 (1276.5s) ETA: 4714min + Val loss: 0.871404 [t_0.0-0.2=1.0695 t_0.2-0.4=0.9982 t_0.4-0.6=0.8427 t_0.6-0.8=0.7097 t_0.8-1.0=0.6330] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 100/299 (33% done) --- + [Epoch 100][10/66] loss=1.025784 avg=0.845193 VRAM=39.5GiB | 33.4% done | ETA(epoch): 1085s + [Epoch 100][20/66] loss=0.874900 avg=0.864274 VRAM=39.2GiB | 33.4% done | ETA(epoch): 890s + [Epoch 100][30/66] loss=0.944206 avg=0.855004 VRAM=39.5GiB | 33.5% done | ETA(epoch): 696s + [Epoch 100][40/66] loss=0.820655 avg=0.858959 VRAM=39.2GiB | 33.5% done | ETA(epoch): 503s + [Epoch 100][50/66] loss=0.647202 avg=0.858043 VRAM=39.5GiB | 33.6% done | ETA(epoch): 310s + [Epoch 100][60/66] loss=0.822056 avg=0.860057 VRAM=39.2GiB | 33.6% done | ETA(epoch): 116s + Train loss: 0.854263 (1278.2s) ETA: 4690min + Val loss: 0.845181 [t_0.0-0.2=1.0581 t_0.2-0.4=1.0153 t_0.4-0.6=0.8416 t_0.6-0.8=0.7047 t_0.8-1.0=0.6414] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 101/299 (34% done) --- + [Epoch 101][10/66] loss=0.775320 avg=0.854177 VRAM=39.5GiB | 33.7% done | ETA(epoch): 1082s + [Epoch 101][20/66] loss=0.742196 avg=0.849684 VRAM=39.2GiB | 33.8% done | ETA(epoch): 888s + [Epoch 101][30/66] loss=0.706474 avg=0.851499 VRAM=39.5GiB | 33.8% done | ETA(epoch): 698s + [Epoch 101][40/66] loss=0.864267 avg=0.848793 VRAM=39.2GiB | 33.9% done | ETA(epoch): 505s + [Epoch 101][50/66] loss=0.820582 avg=0.857605 VRAM=39.5GiB | 33.9% done | ETA(epoch): 311s + [Epoch 101][60/66] loss=0.905570 avg=0.858816 VRAM=39.2GiB | 34.0% done | ETA(epoch): 116s + Train loss: 0.856878 (1279.2s) ETA: 4665min + Val loss: 0.832923 [t_0.0-0.2=1.0705 t_0.2-0.4=1.0263 t_0.4-0.6=0.8352 t_0.6-0.8=0.7137 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 102/299 (34% done) --- + [Epoch 102][10/66] loss=0.868254 avg=0.848096 VRAM=39.5GiB | 34.1% done | ETA(epoch): 1084s + [Epoch 102][20/66] loss=1.035264 avg=0.864484 VRAM=39.2GiB | 34.1% done | ETA(epoch): 889s + [Epoch 102][30/66] loss=0.674951 avg=0.854597 VRAM=39.5GiB | 34.2% done | ETA(epoch): 695s + [Epoch 102][40/66] loss=0.900187 avg=0.859648 VRAM=39.2GiB | 34.2% done | ETA(epoch): 502s + [Epoch 102][50/66] loss=0.974044 avg=0.862661 VRAM=39.5GiB | 34.3% done | ETA(epoch): 309s + [Epoch 102][60/66] loss=0.720046 avg=0.863184 VRAM=39.2GiB | 34.3% done | ETA(epoch): 116s + Train loss: 0.863326 (1273.5s) ETA: 4640min + Val loss: 0.845168 [t_0.0-0.2=1.0630 t_0.2-0.4=1.0247 t_0.4-0.6=0.8499 t_0.6-0.8=0.7260 t_0.8-1.0=0.6492] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 103/299 (34% done) --- + [Epoch 103][10/66] loss=0.879858 avg=0.860502 VRAM=39.5GiB | 34.4% done | ETA(epoch): 1094s + [Epoch 103][20/66] loss=0.762126 avg=0.862405 VRAM=39.2GiB | 34.4% done | ETA(epoch): 894s + [Epoch 103][30/66] loss=0.793277 avg=0.873950 VRAM=39.5GiB | 34.5% done | ETA(epoch): 698s + [Epoch 103][40/66] loss=0.832975 avg=0.863053 VRAM=39.2GiB | 34.5% done | ETA(epoch): 503s + [Epoch 103][50/66] loss=0.959858 avg=0.859702 VRAM=39.5GiB | 34.6% done | ETA(epoch): 310s + [Epoch 103][60/66] loss=0.795004 avg=0.855300 VRAM=39.2GiB | 34.6% done | ETA(epoch): 116s + Train loss: 0.852468 (1276.6s) ETA: 4615min + Val loss: 0.824435 [t_0.0-0.2=1.0656 t_0.2-0.4=0.9676 t_0.4-0.6=0.8254 t_0.6-0.8=0.7358 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 104/299 (35% done) --- + [Epoch 104][10/66] loss=0.833562 avg=0.848227 VRAM=39.5GiB | 34.7% done | ETA(epoch): 1097s + [Epoch 104][20/66] loss=0.801608 avg=0.842996 VRAM=39.2GiB | 34.8% done | ETA(epoch): 896s + [Epoch 104][30/66] loss=0.986105 avg=0.849509 VRAM=39.5GiB | 34.8% done | ETA(epoch): 699s + [Epoch 104][40/66] loss=0.868926 avg=0.851613 VRAM=39.2GiB | 34.9% done | ETA(epoch): 504s + [Epoch 104][50/66] loss=0.825562 avg=0.853888 VRAM=39.5GiB | 34.9% done | ETA(epoch): 310s + [Epoch 104][60/66] loss=0.903888 avg=0.859194 VRAM=39.2GiB | 35.0% done | ETA(epoch): 116s + Train loss: 0.862930 (1281.1s) ETA: 4590min + Val loss: 0.856453 [t_0.0-0.2=1.0529 t_0.2-0.4=0.9982 t_0.4-0.6=0.8773 t_0.6-0.8=0.7147 t_0.8-1.0=0.6582] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 105/299 (35% done) --- + [Epoch 105][10/66] loss=0.820586 avg=0.832342 VRAM=39.5GiB | 35.1% done | ETA(epoch): 1080s + [Epoch 105][20/66] loss=0.968252 avg=0.855178 VRAM=39.2GiB | 35.1% done | ETA(epoch): 887s + [Epoch 105][30/66] loss=0.765009 avg=0.856100 VRAM=39.5GiB | 35.2% done | ETA(epoch): 694s + [Epoch 105][40/66] loss=0.774938 avg=0.848752 VRAM=39.2GiB | 35.2% done | ETA(epoch): 502s + [Epoch 105][50/66] loss=0.900424 avg=0.854207 VRAM=39.5GiB | 35.3% done | ETA(epoch): 309s + [Epoch 105][60/66] loss=0.806311 avg=0.852584 VRAM=39.2GiB | 35.3% done | ETA(epoch): 116s + Train loss: 0.852197 (1278.3s) ETA: 4566min + Val loss: 0.851722 [t_0.0-0.2=1.0706 t_0.2-0.4=0.9911 t_0.4-0.6=0.8617 t_0.6-0.8=0.6953 t_0.8-1.0=0.6530] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 106/299 (35% done) --- + [Epoch 106][10/66] loss=0.910196 avg=0.832559 VRAM=39.5GiB | 35.4% done | ETA(epoch): 1081s + [Epoch 106][20/66] loss=0.834771 avg=0.842869 VRAM=39.2GiB | 35.4% done | ETA(epoch): 888s + [Epoch 106][30/66] loss=0.867419 avg=0.843554 VRAM=39.5GiB | 35.5% done | ETA(epoch): 695s + [Epoch 106][40/66] loss=0.985730 avg=0.855907 VRAM=39.2GiB | 35.5% done | ETA(epoch): 503s + [Epoch 106][50/66] loss=0.939980 avg=0.859572 VRAM=39.5GiB | 35.6% done | ETA(epoch): 310s + [Epoch 106][60/66] loss=0.874147 avg=0.857821 VRAM=39.2GiB | 35.6% done | ETA(epoch): 116s + Train loss: 0.856191 (1279.4s) ETA: 4541min + Val loss: 0.884519 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9997 t_0.4-0.6=0.8786 t_0.6-0.8=0.7094 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 107/299 (36% done) --- + [Epoch 107][10/66] loss=0.768840 avg=0.820927 VRAM=39.5GiB | 35.7% done | ETA(epoch): 1086s + [Epoch 107][20/66] loss=0.886266 avg=0.835005 VRAM=39.2GiB | 35.8% done | ETA(epoch): 890s + [Epoch 107][30/66] loss=0.828829 avg=0.840003 VRAM=39.5GiB | 35.8% done | ETA(epoch): 696s + [Epoch 107][40/66] loss=0.895397 avg=0.841482 VRAM=39.2GiB | 35.9% done | ETA(epoch): 502s + [Epoch 107][50/66] loss=0.869882 avg=0.843124 VRAM=39.5GiB | 35.9% done | ETA(epoch): 309s + [Epoch 107][60/66] loss=0.920228 avg=0.844423 VRAM=39.2GiB | 36.0% done | ETA(epoch): 116s + Train loss: 0.842355 (1275.2s) ETA: 4516min + Val loss: 0.885212 [t_0.0-0.2=1.0864 t_0.2-0.4=0.9770 t_0.4-0.6=0.8500 t_0.6-0.8=0.7114 t_0.8-1.0=0.6468] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 108/299 (36% done) --- + [Epoch 108][10/66] loss=0.817895 avg=0.845365 VRAM=39.5GiB | 36.1% done | ETA(epoch): 1083s + [Epoch 108][20/66] loss=0.884593 avg=0.841395 VRAM=39.2GiB | 36.1% done | ETA(epoch): 891s + [Epoch 108][30/66] loss=0.866821 avg=0.835917 VRAM=39.5GiB | 36.2% done | ETA(epoch): 701s + [Epoch 108][40/66] loss=0.799321 avg=0.839396 VRAM=39.2GiB | 36.2% done | ETA(epoch): 506s + [Epoch 108][50/66] loss=0.747438 avg=0.847318 VRAM=39.5GiB | 36.3% done | ETA(epoch): 311s + [Epoch 108][60/66] loss=0.959695 avg=0.850135 VRAM=39.2GiB | 36.3% done | ETA(epoch): 117s + Train loss: 0.850164 (1282.4s) ETA: 4492min + Val loss: 0.885930 [t_0.0-0.2=1.0672 t_0.2-0.4=1.0027 t_0.4-0.6=0.8383 t_0.6-0.8=0.7156 t_0.8-1.0=0.6593] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 109/299 (36% done) --- + [Epoch 109][10/66] loss=0.739234 avg=0.862269 VRAM=39.5GiB | 36.4% done | ETA(epoch): 1082s + [Epoch 109][20/66] loss=0.854744 avg=0.863354 VRAM=39.2GiB | 36.4% done | ETA(epoch): 897s + [Epoch 109][30/66] loss=0.769203 avg=0.847805 VRAM=39.5GiB | 36.5% done | ETA(epoch): 700s + [Epoch 109][40/66] loss=0.978077 avg=0.857273 VRAM=39.2GiB | 36.5% done | ETA(epoch): 505s + [Epoch 109][50/66] loss=0.755784 avg=0.852421 VRAM=39.5GiB | 36.6% done | ETA(epoch): 310s + [Epoch 109][60/66] loss=1.019602 avg=0.856429 VRAM=39.2GiB | 36.6% done | ETA(epoch): 116s + Train loss: 0.854676 (1278.5s) ETA: 4467min + Val loss: 0.844688 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9779 t_0.4-0.6=0.8085 t_0.6-0.8=0.7173 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 110/299 (37% done) --- + [Epoch 110][10/66] loss=0.851021 avg=0.845952 VRAM=39.5GiB | 36.7% done | ETA(epoch): 1085s + [Epoch 110][20/66] loss=0.767941 avg=0.862010 VRAM=39.2GiB | 36.8% done | ETA(epoch): 896s + [Epoch 110][30/66] loss=0.954221 avg=0.868827 VRAM=39.5GiB | 36.8% done | ETA(epoch): 700s + [Epoch 110][40/66] loss=0.799908 avg=0.863171 VRAM=39.2GiB | 36.9% done | ETA(epoch): 505s + [Epoch 110][50/66] loss=0.826537 avg=0.855200 VRAM=39.5GiB | 36.9% done | ETA(epoch): 310s + [Epoch 110][60/66] loss=0.790536 avg=0.853004 VRAM=39.2GiB | 37.0% done | ETA(epoch): 116s + Train loss: 0.858369 (1278.8s) ETA: 4443min + Val loss: 0.842907 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9787 t_0.4-0.6=0.8640 t_0.6-0.8=0.7185 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 111/299 (37% done) --- + [Epoch 111][10/66] loss=0.770696 avg=0.896518 VRAM=39.5GiB | 37.1% done | ETA(epoch): 1084s + [Epoch 111][20/66] loss=0.819879 avg=0.883278 VRAM=39.2GiB | 37.1% done | ETA(epoch): 889s + [Epoch 111][30/66] loss=0.908920 avg=0.874930 VRAM=39.5GiB | 37.2% done | ETA(epoch): 696s + [Epoch 111][40/66] loss=0.858883 avg=0.864552 VRAM=39.2GiB | 37.2% done | ETA(epoch): 504s + [Epoch 111][50/66] loss=0.922825 avg=0.864623 VRAM=39.5GiB | 37.3% done | ETA(epoch): 310s + [Epoch 111][60/66] loss=0.867178 avg=0.860507 VRAM=39.2GiB | 37.3% done | ETA(epoch): 116s + Train loss: 0.860037 (1277.5s) ETA: 4418min + Val loss: 0.864870 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9956 t_0.4-0.6=0.8351 t_0.6-0.8=0.7221 t_0.8-1.0=0.6457] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 112/299 (37% done) --- + [Epoch 112][10/66] loss=0.899437 avg=0.837013 VRAM=39.5GiB | 37.4% done | ETA(epoch): 1085s + [Epoch 112][20/66] loss=0.903809 avg=0.853474 VRAM=39.2GiB | 37.4% done | ETA(epoch): 894s + [Epoch 112][30/66] loss=0.926926 avg=0.873486 VRAM=39.5GiB | 37.5% done | ETA(epoch): 699s + [Epoch 112][40/66] loss=0.769780 avg=0.867098 VRAM=39.2GiB | 37.5% done | ETA(epoch): 504s + [Epoch 112][50/66] loss=0.779454 avg=0.862824 VRAM=39.5GiB | 37.6% done | ETA(epoch): 310s + [Epoch 112][60/66] loss=0.918315 avg=0.866729 VRAM=39.2GiB | 37.6% done | ETA(epoch): 117s + Train loss: 0.862945 (1282.8s) ETA: 4394min + Val loss: 0.833793 [t_0.0-0.2=1.0759 t_0.2-0.4=0.9997 t_0.4-0.6=0.8433 t_0.6-0.8=0.7256 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 113/299 (38% done) --- + [Epoch 113][10/66] loss=0.768782 avg=0.821119 VRAM=39.5GiB | 37.7% done | ETA(epoch): 1088s + [Epoch 113][20/66] loss=0.907616 avg=0.819156 VRAM=39.2GiB | 37.8% done | ETA(epoch): 893s + [Epoch 113][30/66] loss=0.857812 avg=0.835824 VRAM=39.5GiB | 37.8% done | ETA(epoch): 698s + [Epoch 113][40/66] loss=0.900682 avg=0.838489 VRAM=39.2GiB | 37.9% done | ETA(epoch): 504s + [Epoch 113][50/66] loss=0.932079 avg=0.842545 VRAM=39.5GiB | 37.9% done | ETA(epoch): 310s + [Epoch 113][60/66] loss=0.753801 avg=0.840360 VRAM=39.2GiB | 38.0% done | ETA(epoch): 116s + Train loss: 0.839591 (1279.5s) ETA: 4369min + Val loss: 0.829199 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9928 t_0.4-0.6=0.8199 t_0.6-0.8=0.7160 t_0.8-1.0=0.6611] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 114/299 (38% done) --- + [Epoch 114][10/66] loss=0.816644 avg=0.866519 VRAM=39.5GiB | 38.1% done | ETA(epoch): 1087s + [Epoch 114][20/66] loss=0.803441 avg=0.855322 VRAM=39.2GiB | 38.1% done | ETA(epoch): 893s + [Epoch 114][30/66] loss=0.969295 avg=0.879720 VRAM=39.5GiB | 38.2% done | ETA(epoch): 698s + [Epoch 114][40/66] loss=0.867216 avg=0.874557 VRAM=39.2GiB | 38.2% done | ETA(epoch): 504s + [Epoch 114][50/66] loss=0.977137 avg=0.876524 VRAM=39.5GiB | 38.3% done | ETA(epoch): 310s + [Epoch 114][60/66] loss=0.826432 avg=0.869243 VRAM=39.2GiB | 38.3% done | ETA(epoch): 116s + Train loss: 0.869960 (1276.5s) ETA: 4345min + Val loss: 0.879425 [t_0.0-0.2=1.0562 t_0.2-0.4=1.0143 t_0.4-0.6=0.8440 t_0.6-0.8=0.7130 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 115/299 (38% done) --- + [Epoch 115][10/66] loss=0.912164 avg=0.861734 VRAM=39.5GiB | 38.4% done | ETA(epoch): 1081s + [Epoch 115][20/66] loss=0.847278 avg=0.843739 VRAM=39.2GiB | 38.4% done | ETA(epoch): 888s + [Epoch 115][30/66] loss=1.007700 avg=0.864072 VRAM=39.5GiB | 38.5% done | ETA(epoch): 695s + [Epoch 115][40/66] loss=0.883525 avg=0.859737 VRAM=39.2GiB | 38.5% done | ETA(epoch): 502s + [Epoch 115][50/66] loss=0.773757 avg=0.862244 VRAM=39.5GiB | 38.6% done | ETA(epoch): 309s + [Epoch 115][60/66] loss=0.998009 avg=0.864686 VRAM=39.2GiB | 38.6% done | ETA(epoch): 116s + Train loss: 0.864540 (1277.1s) ETA: 4320min + Val loss: 0.847261 [t_0.0-0.2=1.0698 t_0.2-0.4=1.0110 t_0.4-0.6=0.8683 t_0.6-0.8=0.7072 t_0.8-1.0=0.6409] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 116/299 (39% done) --- + [Epoch 116][10/66] loss=0.800248 avg=0.834629 VRAM=39.5GiB | 38.7% done | ETA(epoch): 1082s + [Epoch 116][20/66] loss=0.923338 avg=0.861273 VRAM=39.2GiB | 38.8% done | ETA(epoch): 888s + [Epoch 116][30/66] loss=0.974704 avg=0.862874 VRAM=39.5GiB | 38.8% done | ETA(epoch): 697s + [Epoch 116][40/66] loss=0.862516 avg=0.860723 VRAM=39.2GiB | 38.9% done | ETA(epoch): 503s + [Epoch 116][50/66] loss=1.027128 avg=0.859279 VRAM=39.5GiB | 38.9% done | ETA(epoch): 310s + [Epoch 116][60/66] loss=0.777867 avg=0.852480 VRAM=39.2GiB | 39.0% done | ETA(epoch): 116s + Train loss: 0.855385 (1279.7s) ETA: 4296min + Val loss: 0.844873 [t_0.0-0.2=1.0598 t_0.2-0.4=1.0124 t_0.4-0.6=0.8195 t_0.6-0.8=0.7065 t_0.8-1.0=0.6415] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 117/299 (39% done) --- + [Epoch 117][10/66] loss=0.950091 avg=0.906999 VRAM=39.5GiB | 39.1% done | ETA(epoch): 1082s + [Epoch 117][20/66] loss=0.910681 avg=0.893185 VRAM=39.2GiB | 39.1% done | ETA(epoch): 888s + [Epoch 117][30/66] loss=0.789095 avg=0.872670 VRAM=39.5GiB | 39.2% done | ETA(epoch): 695s + [Epoch 117][40/66] loss=0.980654 avg=0.868987 VRAM=39.2GiB | 39.2% done | ETA(epoch): 502s + [Epoch 117][50/66] loss=0.818985 avg=0.864608 VRAM=39.5GiB | 39.3% done | ETA(epoch): 309s + [Epoch 117][60/66] loss=0.792332 avg=0.858497 VRAM=39.2GiB | 39.3% done | ETA(epoch): 116s + Train loss: 0.857352 (1273.2s) ETA: 4271min + Val loss: 0.822695 [t_0.0-0.2=1.0659 t_0.2-0.4=0.9871 t_0.4-0.6=0.8325 t_0.6-0.8=0.6932 t_0.8-1.0=0.6453] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 118/299 (39% done) --- + [Epoch 118][10/66] loss=0.748190 avg=0.839714 VRAM=39.5GiB | 39.4% done | ETA(epoch): 1082s + [Epoch 118][20/66] loss=0.777671 avg=0.841643 VRAM=39.2GiB | 39.4% done | ETA(epoch): 888s + [Epoch 118][30/66] loss=0.837785 avg=0.840121 VRAM=39.5GiB | 39.5% done | ETA(epoch): 695s + [Epoch 118][40/66] loss=0.890987 avg=0.852008 VRAM=39.2GiB | 39.5% done | ETA(epoch): 502s + [Epoch 118][50/66] loss=0.950033 avg=0.855246 VRAM=39.5GiB | 39.6% done | ETA(epoch): 309s + [Epoch 118][60/66] loss=0.881767 avg=0.856020 VRAM=39.2GiB | 39.6% done | ETA(epoch): 116s + Train loss: 0.859622 (1275.6s) ETA: 4247min + Val loss: 0.849123 [t_0.0-0.2=1.0720 t_0.2-0.4=0.9903 t_0.4-0.6=0.8307 t_0.6-0.8=0.7017 t_0.8-1.0=0.6592] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 119/299 (40% done) --- + [Epoch 119][10/66] loss=0.896260 avg=0.822562 VRAM=39.5GiB | 39.7% done | ETA(epoch): 1082s + [Epoch 119][20/66] loss=0.863344 avg=0.838860 VRAM=39.2GiB | 39.8% done | ETA(epoch): 888s + [Epoch 119][30/66] loss=0.750511 avg=0.850953 VRAM=39.5GiB | 39.8% done | ETA(epoch): 695s + [Epoch 119][40/66] loss=0.795356 avg=0.856700 VRAM=39.2GiB | 39.9% done | ETA(epoch): 502s + [Epoch 119][50/66] loss=0.840784 avg=0.863976 VRAM=39.5GiB | 39.9% done | ETA(epoch): 309s + [Epoch 119][60/66] loss=0.798144 avg=0.859615 VRAM=39.2GiB | 40.0% done | ETA(epoch): 116s + Train loss: 0.858978 (1274.3s) ETA: 4223min + Val loss: 0.847357 [t_0.0-0.2=1.0656 t_0.2-0.4=0.9902 t_0.4-0.6=0.8269 t_0.6-0.8=0.7128 t_0.8-1.0=0.6569] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 120/299 (40% done) --- + [MilestoneVis] train_0 step 7920 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 7920 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 7920 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 7920 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 7920 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 7920 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 7920 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 7920 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 7920 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 7920 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + [Epoch 120][10/66] loss=0.812778 avg=0.814341 VRAM=39.5GiB | 40.1% done | ETA(epoch): 6827s + [Epoch 120][20/66] loss=0.710725 avg=0.828269 VRAM=39.2GiB | 40.1% done | ETA(epoch): 3248s + [Epoch 120][30/66] loss=0.925097 avg=0.856466 VRAM=39.5GiB | 40.2% done | ETA(epoch): 1927s + [Epoch 120][40/66] loss=0.799441 avg=0.844691 VRAM=39.2GiB | 40.2% done | ETA(epoch): 1170s + [Epoch 120][50/66] loss=0.827629 avg=0.849095 VRAM=39.5GiB | 40.3% done | ETA(epoch): 638s + [Epoch 120][60/66] loss=0.842040 avg=0.858207 VRAM=39.2GiB | 40.3% done | ETA(epoch): 219s + Train loss: 0.859886 (2306.1s) ETA: 4224min + Val loss: 0.836704 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0101 t_0.4-0.6=0.8180 t_0.6-0.8=0.7123 t_0.8-1.0=0.6308] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 121/299 (40% done) --- + [Epoch 121][10/66] loss=0.848483 avg=0.845994 VRAM=39.5GiB | 40.4% done | ETA(epoch): 1082s + [Epoch 121][20/66] loss=0.846285 avg=0.846727 VRAM=39.2GiB | 40.4% done | ETA(epoch): 887s + [Epoch 121][30/66] loss=0.729698 avg=0.847899 VRAM=39.5GiB | 40.5% done | ETA(epoch): 694s + [Epoch 121][40/66] loss=0.794799 avg=0.853580 VRAM=39.2GiB | 40.5% done | ETA(epoch): 501s + [Epoch 121][50/66] loss=0.816941 avg=0.854658 VRAM=39.5GiB | 40.6% done | ETA(epoch): 308s + [Epoch 121][60/66] loss=0.859234 avg=0.850405 VRAM=39.2GiB | 40.6% done | ETA(epoch): 116s + Train loss: 0.852326 (1271.4s) ETA: 4199min + Val loss: 0.865558 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9895 t_0.4-0.6=0.8464 t_0.6-0.8=0.7263 t_0.8-1.0=0.6572] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 122/299 (41% done) --- + [Epoch 122][10/66] loss=0.769245 avg=0.809561 VRAM=39.5GiB | 40.7% done | ETA(epoch): 1077s + [Epoch 122][20/66] loss=0.800891 avg=0.822762 VRAM=39.2GiB | 40.8% done | ETA(epoch): 886s + [Epoch 122][30/66] loss=0.852112 avg=0.832151 VRAM=39.5GiB | 40.8% done | ETA(epoch): 693s + [Epoch 122][40/66] loss=0.811443 avg=0.835626 VRAM=39.2GiB | 40.9% done | ETA(epoch): 503s + [Epoch 122][50/66] loss=0.842731 avg=0.834299 VRAM=39.5GiB | 40.9% done | ETA(epoch): 309s + [Epoch 122][60/66] loss=0.747565 avg=0.830941 VRAM=39.2GiB | 41.0% done | ETA(epoch): 116s + Train loss: 0.830393 (1274.2s) ETA: 4174min + Val loss: 0.840544 [t_0.0-0.2=1.0918 t_0.2-0.4=1.0037 t_0.4-0.6=0.8598 t_0.6-0.8=0.7281 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 123/299 (41% done) --- + [Epoch 123][10/66] loss=0.858541 avg=0.830979 VRAM=39.5GiB | 41.1% done | ETA(epoch): 1077s + [Epoch 123][20/66] loss=0.805896 avg=0.850049 VRAM=39.2GiB | 41.1% done | ETA(epoch): 885s + [Epoch 123][30/66] loss=0.949743 avg=0.856432 VRAM=39.5GiB | 41.2% done | ETA(epoch): 693s + [Epoch 123][40/66] loss=0.902195 avg=0.849754 VRAM=39.2GiB | 41.2% done | ETA(epoch): 502s + [Epoch 123][50/66] loss=0.765037 avg=0.846058 VRAM=39.5GiB | 41.3% done | ETA(epoch): 309s + [Epoch 123][60/66] loss=0.832760 avg=0.852745 VRAM=39.2GiB | 41.3% done | ETA(epoch): 116s + Train loss: 0.855324 (1272.8s) ETA: 4150min + Val loss: 0.870806 [t_0.0-0.2=1.0665 t_0.2-0.4=0.9730 t_0.4-0.6=0.8535 t_0.6-0.8=0.6977 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 124/299 (41% done) --- + [Epoch 124][10/66] loss=0.911565 avg=0.869711 VRAM=39.5GiB | 41.4% done | ETA(epoch): 1078s + [Epoch 124][20/66] loss=0.827236 avg=0.872202 VRAM=39.2GiB | 41.4% done | ETA(epoch): 886s + [Epoch 124][30/66] loss=0.976562 avg=0.874131 VRAM=39.5GiB | 41.5% done | ETA(epoch): 696s + [Epoch 124][40/66] loss=0.834480 avg=0.870642 VRAM=39.2GiB | 41.5% done | ETA(epoch): 502s + [Epoch 124][50/66] loss=0.817462 avg=0.861691 VRAM=39.5GiB | 41.6% done | ETA(epoch): 309s + [Epoch 124][60/66] loss=0.891169 avg=0.859793 VRAM=39.2GiB | 41.6% done | ETA(epoch): 116s + Train loss: 0.852435 (1273.2s) ETA: 4125min + Val loss: 0.828668 [t_0.0-0.2=1.0523 t_0.2-0.4=0.9948 t_0.4-0.6=0.8388 t_0.6-0.8=0.6979 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 125/299 (42% done) --- + [Epoch 125][10/66] loss=0.894559 avg=0.858339 VRAM=39.5GiB | 41.7% done | ETA(epoch): 1077s + [Epoch 125][20/66] loss=0.805060 avg=0.859581 VRAM=39.2GiB | 41.8% done | ETA(epoch): 885s + [Epoch 125][30/66] loss=0.950671 avg=0.852374 VRAM=39.5GiB | 41.8% done | ETA(epoch): 692s + [Epoch 125][40/66] loss=0.765029 avg=0.857124 VRAM=39.2GiB | 41.9% done | ETA(epoch): 500s + [Epoch 125][50/66] loss=0.795735 avg=0.859911 VRAM=39.5GiB | 41.9% done | ETA(epoch): 308s + [Epoch 125][60/66] loss=0.793070 avg=0.854508 VRAM=39.2GiB | 42.0% done | ETA(epoch): 116s + Train loss: 0.857425 (1272.0s) ETA: 4100min + Val loss: 0.857464 [t_0.0-0.2=1.0658 t_0.2-0.4=1.0147 t_0.4-0.6=0.8554 t_0.6-0.8=0.6965 t_0.8-1.0=0.6521] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 126/299 (42% done) --- + [Epoch 126][10/66] loss=0.883101 avg=0.891020 VRAM=39.5GiB | 42.1% done | ETA(epoch): 1076s + [Epoch 126][20/66] loss=0.888898 avg=0.867130 VRAM=39.2GiB | 42.1% done | ETA(epoch): 883s + [Epoch 126][30/66] loss=0.848584 avg=0.869819 VRAM=39.5GiB | 42.2% done | ETA(epoch): 693s + [Epoch 126][40/66] loss=0.881655 avg=0.875999 VRAM=39.2GiB | 42.2% done | ETA(epoch): 501s + [Epoch 126][50/66] loss=0.971030 avg=0.877353 VRAM=39.5GiB | 42.3% done | ETA(epoch): 308s + [Epoch 126][60/66] loss=0.788856 avg=0.872862 VRAM=39.2GiB | 42.3% done | ETA(epoch): 116s + Train loss: 0.871078 (1272.7s) ETA: 4076min + Val loss: 0.812318 [t_0.0-0.2=1.0807 t_0.2-0.4=0.9981 t_0.4-0.6=0.8294 t_0.6-0.8=0.6996 t_0.8-1.0=0.6264] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0126 (BEST) + Deleted old checkpoint: checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 19.3/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 127/299 (42% done) --- + [Epoch 127][10/66] loss=0.760416 avg=0.804897 VRAM=39.5GiB | 42.4% done | ETA(epoch): 1082s + [Epoch 127][20/66] loss=0.815645 avg=0.823280 VRAM=39.2GiB | 42.4% done | ETA(epoch): 890s + [Epoch 127][30/66] loss=0.942624 avg=0.832902 VRAM=39.5GiB | 42.5% done | ETA(epoch): 695s + [Epoch 127][40/66] loss=0.795128 avg=0.831990 VRAM=39.2GiB | 42.5% done | ETA(epoch): 502s + [Epoch 127][50/66] loss=0.821887 avg=0.835156 VRAM=39.5GiB | 42.6% done | ETA(epoch): 309s + [Epoch 127][60/66] loss=0.884963 avg=0.838862 VRAM=39.2GiB | 42.6% done | ETA(epoch): 116s + Train loss: 0.840552 (1274.6s) ETA: 4051min + Val loss: 0.862267 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9956 t_0.4-0.6=0.8304 t_0.6-0.8=0.6979 t_0.8-1.0=0.6310] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 128/299 (43% done) --- + [Epoch 128][10/66] loss=0.761393 avg=0.838562 VRAM=39.5GiB | 42.7% done | ETA(epoch): 1084s + [Epoch 128][20/66] loss=0.708924 avg=0.853224 VRAM=39.2GiB | 42.8% done | ETA(epoch): 892s + [Epoch 128][30/66] loss=0.923927 avg=0.862409 VRAM=39.5GiB | 42.8% done | ETA(epoch): 698s + [Epoch 128][40/66] loss=0.901352 avg=0.860001 VRAM=39.2GiB | 42.9% done | ETA(epoch): 504s + [Epoch 128][50/66] loss=0.949508 avg=0.856934 VRAM=39.5GiB | 42.9% done | ETA(epoch): 311s + [Epoch 128][60/66] loss=0.885148 avg=0.848583 VRAM=39.2GiB | 43.0% done | ETA(epoch): 117s + Train loss: 0.845587 (1281.7s) ETA: 4027min + Val loss: 0.851056 [t_0.0-0.2=1.0830 t_0.2-0.4=0.9986 t_0.4-0.6=0.8223 t_0.6-0.8=0.6885 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 129/299 (43% done) --- + [Epoch 129][10/66] loss=0.713302 avg=0.884753 VRAM=39.5GiB | 43.1% done | ETA(epoch): 1087s + [Epoch 129][20/66] loss=0.812495 avg=0.885193 VRAM=39.2GiB | 43.1% done | ETA(epoch): 893s + [Epoch 129][30/66] loss=0.959262 avg=0.878210 VRAM=39.5GiB | 43.2% done | ETA(epoch): 698s + [Epoch 129][40/66] loss=0.774044 avg=0.868110 VRAM=39.2GiB | 43.2% done | ETA(epoch): 503s + [Epoch 129][50/66] loss=0.925799 avg=0.865796 VRAM=39.5GiB | 43.3% done | ETA(epoch): 309s + [Epoch 129][60/66] loss=0.839684 avg=0.869070 VRAM=39.2GiB | 43.3% done | ETA(epoch): 116s + Train loss: 0.869567 (1274.9s) ETA: 4002min + Val loss: 0.831717 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9913 t_0.4-0.6=0.8353 t_0.6-0.8=0.7175 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0129 +[MEM @ epoch 129 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 130/299 (43% done) --- + [Epoch 130][10/66] loss=1.015774 avg=0.870322 VRAM=39.5GiB | 43.4% done | ETA(epoch): 1081s + [Epoch 130][20/66] loss=0.833009 avg=0.863673 VRAM=39.2GiB | 43.4% done | ETA(epoch): 887s + [Epoch 130][30/66] loss=0.819698 avg=0.869679 VRAM=39.5GiB | 43.5% done | ETA(epoch): 694s + [Epoch 130][40/66] loss=0.810583 avg=0.859975 VRAM=39.2GiB | 43.5% done | ETA(epoch): 503s + [Epoch 130][50/66] loss=0.840490 avg=0.862708 VRAM=39.5GiB | 43.6% done | ETA(epoch): 310s + [Epoch 130][60/66] loss=0.922956 avg=0.863372 VRAM=39.2GiB | 43.6% done | ETA(epoch): 116s + Train loss: 0.862148 (1276.1s) ETA: 3978min + Val loss: 0.825904 [t_0.0-0.2=1.0656 t_0.2-0.4=1.0247 t_0.4-0.6=0.8318 t_0.6-0.8=0.7245 t_0.8-1.0=0.6504] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 131/299 (44% done) --- + [Epoch 131][10/66] loss=0.852848 avg=0.868767 VRAM=39.5GiB | 43.7% done | ETA(epoch): 1086s + [Epoch 131][20/66] loss=0.991185 avg=0.864648 VRAM=39.2GiB | 43.8% done | ETA(epoch): 893s + [Epoch 131][30/66] loss=0.868278 avg=0.869916 VRAM=39.5GiB | 43.8% done | ETA(epoch): 699s + [Epoch 131][40/66] loss=0.775560 avg=0.854926 VRAM=39.2GiB | 43.9% done | ETA(epoch): 504s + [Epoch 131][50/66] loss=0.875193 avg=0.854523 VRAM=39.5GiB | 43.9% done | ETA(epoch): 310s + [Epoch 131][60/66] loss=0.927777 avg=0.859348 VRAM=39.2GiB | 44.0% done | ETA(epoch): 116s + Train loss: 0.854223 (1277.0s) ETA: 3953min + Val loss: 0.851525 [t_0.0-0.2=1.0589 t_0.2-0.4=1.0431 t_0.4-0.6=0.8416 t_0.6-0.8=0.6947 t_0.8-1.0=0.6375] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 132/299 (44% done) --- + [Epoch 132][10/66] loss=0.878783 avg=0.878987 VRAM=39.5GiB | 44.1% done | ETA(epoch): 1084s + [Epoch 132][20/66] loss=0.733000 avg=0.870193 VRAM=39.2GiB | 44.1% done | ETA(epoch): 901s + [Epoch 132][30/66] loss=0.745695 avg=0.855458 VRAM=39.5GiB | 44.2% done | ETA(epoch): 703s + [Epoch 132][40/66] loss=0.738204 avg=0.862494 VRAM=39.2GiB | 44.2% done | ETA(epoch): 507s + [Epoch 132][50/66] loss=0.928728 avg=0.862733 VRAM=39.5GiB | 44.3% done | ETA(epoch): 311s + [Epoch 132][60/66] loss=0.848006 avg=0.857938 VRAM=39.2GiB | 44.3% done | ETA(epoch): 117s + Train loss: 0.858877 (1282.5s) ETA: 3929min + Val loss: 0.861431 [t_0.0-0.2=1.0638 t_0.2-0.4=0.9651 t_0.4-0.6=0.8348 t_0.6-0.8=0.6838 t_0.8-1.0=0.6492] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 133/299 (44% done) --- + [Epoch 133][10/66] loss=0.737842 avg=0.876835 VRAM=39.5GiB | 44.4% done | ETA(epoch): 1095s + [Epoch 133][20/66] loss=0.765952 avg=0.854072 VRAM=39.2GiB | 44.4% done | ETA(epoch): 894s + [Epoch 133][30/66] loss=0.792300 avg=0.849006 VRAM=39.5GiB | 44.5% done | ETA(epoch): 697s + [Epoch 133][40/66] loss=0.792454 avg=0.853738 VRAM=39.2GiB | 44.5% done | ETA(epoch): 504s + [Epoch 133][50/66] loss=0.866525 avg=0.854191 VRAM=39.5GiB | 44.6% done | ETA(epoch): 310s + [Epoch 133][60/66] loss=0.951093 avg=0.853053 VRAM=39.2GiB | 44.6% done | ETA(epoch): 116s + Train loss: 0.857364 (1277.1s) ETA: 3905min + Val loss: 0.859706 [t_0.0-0.2=1.0787 t_0.2-0.4=1.0238 t_0.4-0.6=0.8228 t_0.6-0.8=0.7193 t_0.8-1.0=0.6422] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 134/299 (45% done) --- + [Epoch 134][10/66] loss=0.778543 avg=0.858451 VRAM=39.5GiB | 44.7% done | ETA(epoch): 1085s + [Epoch 134][20/66] loss=0.805972 avg=0.859653 VRAM=39.2GiB | 44.8% done | ETA(epoch): 889s + [Epoch 134][30/66] loss=0.730383 avg=0.849175 VRAM=39.5GiB | 44.8% done | ETA(epoch): 695s + [Epoch 134][40/66] loss=0.789630 avg=0.845262 VRAM=39.2GiB | 44.9% done | ETA(epoch): 501s + [Epoch 134][50/66] loss=0.837420 avg=0.844050 VRAM=39.5GiB | 44.9% done | ETA(epoch): 309s + [Epoch 134][60/66] loss=0.769195 avg=0.849073 VRAM=39.2GiB | 45.0% done | ETA(epoch): 116s + Train loss: 0.848632 (1274.3s) ETA: 3880min + Val loss: 0.870724 [t_0.0-0.2=1.0591 t_0.2-0.4=1.0139 t_0.4-0.6=0.8958 t_0.6-0.8=0.7028 t_0.8-1.0=0.6594] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 135/299 (45% done) --- + [Epoch 135][10/66] loss=0.925500 avg=0.851168 VRAM=39.5GiB | 45.1% done | ETA(epoch): 1083s + [Epoch 135][20/66] loss=1.059431 avg=0.855745 VRAM=39.2GiB | 45.1% done | ETA(epoch): 889s + [Epoch 135][30/66] loss=0.784646 avg=0.833507 VRAM=39.5GiB | 45.2% done | ETA(epoch): 696s + [Epoch 135][40/66] loss=0.951391 avg=0.843885 VRAM=39.2GiB | 45.2% done | ETA(epoch): 502s + [Epoch 135][50/66] loss=0.899740 avg=0.847579 VRAM=39.5GiB | 45.3% done | ETA(epoch): 309s + [Epoch 135][60/66] loss=0.939302 avg=0.849419 VRAM=39.2GiB | 45.3% done | ETA(epoch): 116s + Train loss: 0.850241 (1273.5s) ETA: 3856min + Val loss: 0.859065 [t_0.0-0.2=1.0839 t_0.2-0.4=1.0084 t_0.4-0.6=0.8749 t_0.6-0.8=0.6770 t_0.8-1.0=0.6372] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 136/299 (45% done) --- + [Epoch 136][10/66] loss=0.958938 avg=0.849980 VRAM=39.5GiB | 45.4% done | ETA(epoch): 1087s + [Epoch 136][20/66] loss=0.793637 avg=0.856259 VRAM=39.2GiB | 45.4% done | ETA(epoch): 892s + [Epoch 136][30/66] loss=0.850688 avg=0.848818 VRAM=39.5GiB | 45.5% done | ETA(epoch): 697s + [Epoch 136][40/66] loss=0.917477 avg=0.842030 VRAM=39.2GiB | 45.5% done | ETA(epoch): 503s + [Epoch 136][50/66] loss=0.878251 avg=0.838220 VRAM=39.5GiB | 45.6% done | ETA(epoch): 310s + [Epoch 136][60/66] loss=0.867882 avg=0.844453 VRAM=39.2GiB | 45.6% done | ETA(epoch): 116s + Train loss: 0.842985 (1278.5s) ETA: 3832min + Val loss: 0.839812 [t_0.0-0.2=1.0519 t_0.2-0.4=1.0126 t_0.4-0.6=0.8376 t_0.6-0.8=0.7201 t_0.8-1.0=0.6430] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 137/299 (46% done) --- + [Epoch 137][10/66] loss=0.817385 avg=0.877044 VRAM=39.5GiB | 45.7% done | ETA(epoch): 1086s + [Epoch 137][20/66] loss=0.858723 avg=0.856410 VRAM=39.2GiB | 45.8% done | ETA(epoch): 891s + [Epoch 137][30/66] loss=0.759963 avg=0.839115 VRAM=39.5GiB | 45.8% done | ETA(epoch): 697s + [Epoch 137][40/66] loss=0.768892 avg=0.840239 VRAM=39.2GiB | 45.9% done | ETA(epoch): 503s + [Epoch 137][50/66] loss=0.981456 avg=0.847981 VRAM=39.5GiB | 45.9% done | ETA(epoch): 310s + [Epoch 137][60/66] loss=0.920947 avg=0.855625 VRAM=39.2GiB | 46.0% done | ETA(epoch): 116s + Train loss: 0.857612 (1276.5s) ETA: 3808min + Val loss: 0.872983 [t_0.0-0.2=1.0757 t_0.2-0.4=1.0037 t_0.4-0.6=0.8511 t_0.6-0.8=0.6985 t_0.8-1.0=0.6339] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 138/299 (46% done) --- + [Epoch 138][10/66] loss=0.819122 avg=0.842463 VRAM=39.5GiB | 46.1% done | ETA(epoch): 1081s + [Epoch 138][20/66] loss=0.879450 avg=0.869280 VRAM=39.2GiB | 46.1% done | ETA(epoch): 888s + [Epoch 138][30/66] loss=0.703035 avg=0.856875 VRAM=39.5GiB | 46.2% done | ETA(epoch): 694s + [Epoch 138][40/66] loss=0.868556 avg=0.857582 VRAM=39.2GiB | 46.2% done | ETA(epoch): 501s + [Epoch 138][50/66] loss=0.861929 avg=0.856287 VRAM=39.5GiB | 46.3% done | ETA(epoch): 309s + [Epoch 138][60/66] loss=0.845573 avg=0.857150 VRAM=39.2GiB | 46.3% done | ETA(epoch): 116s + Train loss: 0.858452 (1274.7s) ETA: 3783min + Val loss: 0.879816 [t_0.0-0.2=1.0591 t_0.2-0.4=1.0223 t_0.4-0.6=0.8406 t_0.6-0.8=0.7232 t_0.8-1.0=0.6353] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 139/299 (46% done) --- + [Epoch 139][10/66] loss=0.929651 avg=0.856686 VRAM=39.5GiB | 46.4% done | ETA(epoch): 1082s + [Epoch 139][20/66] loss=0.837987 avg=0.838486 VRAM=39.2GiB | 46.4% done | ETA(epoch): 890s + [Epoch 139][30/66] loss=0.952317 avg=0.858679 VRAM=39.5GiB | 46.5% done | ETA(epoch): 697s + [Epoch 139][40/66] loss=0.825006 avg=0.863663 VRAM=39.2GiB | 46.5% done | ETA(epoch): 503s + [Epoch 139][50/66] loss=1.057471 avg=0.862459 VRAM=39.5GiB | 46.6% done | ETA(epoch): 309s + [Epoch 139][60/66] loss=0.794889 avg=0.850346 VRAM=39.2GiB | 46.6% done | ETA(epoch): 116s + Train loss: 0.849713 (1275.2s) ETA: 3759min + Val loss: 0.876659 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9818 t_0.4-0.6=0.8449 t_0.6-0.8=0.7357 t_0.8-1.0=0.6399] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 140/299 (47% done) --- + [Epoch 140][10/66] loss=0.939127 avg=0.876707 VRAM=39.5GiB | 46.7% done | ETA(epoch): 1089s + [Epoch 140][20/66] loss=0.825408 avg=0.870479 VRAM=39.2GiB | 46.8% done | ETA(epoch): 891s + [Epoch 140][30/66] loss=0.881210 avg=0.867465 VRAM=39.5GiB | 46.8% done | ETA(epoch): 697s + [Epoch 140][40/66] loss=0.935178 avg=0.867766 VRAM=39.2GiB | 46.9% done | ETA(epoch): 503s + [Epoch 140][50/66] loss=0.843347 avg=0.857120 VRAM=39.5GiB | 46.9% done | ETA(epoch): 309s + [Epoch 140][60/66] loss=0.876654 avg=0.857786 VRAM=39.2GiB | 47.0% done | ETA(epoch): 116s + Train loss: 0.855766 (1279.1s) ETA: 3735min + Val loss: 0.874377 [t_0.0-0.2=1.0668 t_0.2-0.4=0.9866 t_0.4-0.6=0.8342 t_0.6-0.8=0.7219 t_0.8-1.0=0.6463] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 141/299 (47% done) --- + [Epoch 141][10/66] loss=0.736683 avg=0.837849 VRAM=39.5GiB | 47.1% done | ETA(epoch): 1099s + [Epoch 141][20/66] loss=0.832421 avg=0.836186 VRAM=39.2GiB | 47.1% done | ETA(epoch): 897s + [Epoch 141][30/66] loss=0.920000 avg=0.849829 VRAM=39.5GiB | 47.2% done | ETA(epoch): 701s + [Epoch 141][40/66] loss=0.891480 avg=0.849596 VRAM=39.2GiB | 47.2% done | ETA(epoch): 506s + [Epoch 141][50/66] loss=0.896859 avg=0.845309 VRAM=39.5GiB | 47.3% done | ETA(epoch): 311s + [Epoch 141][60/66] loss=0.852889 avg=0.852356 VRAM=39.2GiB | 47.3% done | ETA(epoch): 116s + Train loss: 0.853941 (1280.4s) ETA: 3711min + Val loss: 0.830884 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9456 t_0.4-0.6=0.8604 t_0.6-0.8=0.7123 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 142/299 (47% done) --- + [Epoch 142][10/66] loss=0.908024 avg=0.878524 VRAM=39.5GiB | 47.4% done | ETA(epoch): 1085s + [Epoch 142][20/66] loss=0.796635 avg=0.878980 VRAM=39.2GiB | 47.4% done | ETA(epoch): 892s + [Epoch 142][30/66] loss=0.786675 avg=0.864226 VRAM=39.5GiB | 47.5% done | ETA(epoch): 699s + [Epoch 142][40/66] loss=0.859310 avg=0.854966 VRAM=39.2GiB | 47.5% done | ETA(epoch): 504s + [Epoch 142][50/66] loss=0.987918 avg=0.854892 VRAM=39.5GiB | 47.6% done | ETA(epoch): 310s + [Epoch 142][60/66] loss=0.836542 avg=0.860266 VRAM=39.2GiB | 47.6% done | ETA(epoch): 117s + Train loss: 0.859643 (1283.2s) ETA: 3687min + Val loss: 0.874068 [t_0.0-0.2=1.0653 t_0.2-0.4=0.9866 t_0.4-0.6=0.8682 t_0.6-0.8=0.6929 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 143/299 (48% done) --- + [Epoch 143][10/66] loss=0.836727 avg=0.882954 VRAM=39.5GiB | 47.7% done | ETA(epoch): 1083s + [Epoch 143][20/66] loss=0.775009 avg=0.840377 VRAM=39.2GiB | 47.8% done | ETA(epoch): 889s + [Epoch 143][30/66] loss=0.849576 avg=0.837005 VRAM=39.5GiB | 47.8% done | ETA(epoch): 696s + [Epoch 143][40/66] loss=0.693479 avg=0.831931 VRAM=39.2GiB | 47.9% done | ETA(epoch): 503s + [Epoch 143][50/66] loss=0.941280 avg=0.835379 VRAM=39.5GiB | 47.9% done | ETA(epoch): 311s + [Epoch 143][60/66] loss=0.902019 avg=0.837610 VRAM=39.2GiB | 48.0% done | ETA(epoch): 117s + Train loss: 0.841457 (1282.0s) ETA: 3663min + Val loss: 0.859216 [t_0.0-0.2=1.0841 t_0.2-0.4=1.0261 t_0.4-0.6=0.8276 t_0.6-0.8=0.6929 t_0.8-1.0=0.6313] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 144/299 (48% done) --- + [Epoch 144][10/66] loss=0.936475 avg=0.835271 VRAM=39.5GiB | 48.1% done | ETA(epoch): 1085s + [Epoch 144][20/66] loss=0.703540 avg=0.843711 VRAM=39.2GiB | 48.1% done | ETA(epoch): 890s + [Epoch 144][30/66] loss=0.892890 avg=0.849396 VRAM=39.5GiB | 48.2% done | ETA(epoch): 696s + [Epoch 144][40/66] loss=0.869726 avg=0.846840 VRAM=39.2GiB | 48.2% done | ETA(epoch): 503s + [Epoch 144][50/66] loss=0.724966 avg=0.842919 VRAM=39.5GiB | 48.3% done | ETA(epoch): 310s + [Epoch 144][60/66] loss=0.778505 avg=0.851025 VRAM=39.2GiB | 48.3% done | ETA(epoch): 116s + Train loss: 0.850612 (1276.3s) ETA: 3639min + Val loss: 0.846563 [t_0.0-0.2=1.0746 t_0.2-0.4=0.9744 t_0.4-0.6=0.8553 t_0.6-0.8=0.7228 t_0.8-1.0=0.6452] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 145/299 (48% done) --- + [Epoch 145][10/66] loss=0.781630 avg=0.841367 VRAM=39.5GiB | 48.4% done | ETA(epoch): 1087s + [Epoch 145][20/66] loss=0.777190 avg=0.846035 VRAM=39.2GiB | 48.4% done | ETA(epoch): 891s + [Epoch 145][30/66] loss=0.941477 avg=0.846815 VRAM=39.5GiB | 48.5% done | ETA(epoch): 702s + [Epoch 145][40/66] loss=0.852836 avg=0.849200 VRAM=39.2GiB | 48.5% done | ETA(epoch): 506s + [Epoch 145][50/66] loss=0.832697 avg=0.846483 VRAM=39.5GiB | 48.6% done | ETA(epoch): 311s + [Epoch 145][60/66] loss=0.873537 avg=0.851165 VRAM=39.2GiB | 48.6% done | ETA(epoch): 117s + Train loss: 0.849730 (1281.2s) ETA: 3614min + Val loss: 0.852161 [t_0.0-0.2=1.0505 t_0.2-0.4=1.0396 t_0.4-0.6=0.8275 t_0.6-0.8=0.7253 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 146/299 (49% done) --- + [Epoch 146][10/66] loss=0.844005 avg=0.827028 VRAM=39.5GiB | 48.7% done | ETA(epoch): 1083s + [Epoch 146][20/66] loss=0.972376 avg=0.839016 VRAM=39.2GiB | 48.8% done | ETA(epoch): 890s + [Epoch 146][30/66] loss=0.894898 avg=0.862556 VRAM=39.5GiB | 48.8% done | ETA(epoch): 697s + [Epoch 146][40/66] loss=0.789617 avg=0.854549 VRAM=39.2GiB | 48.9% done | ETA(epoch): 503s + [Epoch 146][50/66] loss=0.824223 avg=0.865406 VRAM=39.5GiB | 48.9% done | ETA(epoch): 310s + [Epoch 146][60/66] loss=0.845963 avg=0.862653 VRAM=39.2GiB | 49.0% done | ETA(epoch): 116s + Train loss: 0.861441 (1277.3s) ETA: 3590min + Val loss: 0.859748 [t_0.0-0.2=1.0836 t_0.2-0.4=1.0024 t_0.4-0.6=0.8250 t_0.6-0.8=0.7080 t_0.8-1.0=0.6415] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 147/299 (49% done) --- + [Epoch 147][10/66] loss=0.856507 avg=0.865521 VRAM=39.5GiB | 49.1% done | ETA(epoch): 1085s + [Epoch 147][20/66] loss=0.966334 avg=0.846399 VRAM=39.2GiB | 49.1% done | ETA(epoch): 895s + [Epoch 147][30/66] loss=0.905331 avg=0.848365 VRAM=39.5GiB | 49.2% done | ETA(epoch): 699s + [Epoch 147][40/66] loss=0.739986 avg=0.843208 VRAM=39.2GiB | 49.2% done | ETA(epoch): 504s + [Epoch 147][50/66] loss=0.856959 avg=0.855928 VRAM=39.5GiB | 49.3% done | ETA(epoch): 310s + [Epoch 147][60/66] loss=0.827486 avg=0.851659 VRAM=39.2GiB | 49.3% done | ETA(epoch): 116s + Train loss: 0.848799 (1278.7s) ETA: 3566min + Val loss: 0.866200 [t_0.0-0.2=1.0614 t_0.2-0.4=0.9628 t_0.4-0.6=0.8256 t_0.6-0.8=0.7308 t_0.8-1.0=0.6337] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 148/299 (49% done) --- + [Epoch 148][10/66] loss=0.960372 avg=0.838151 VRAM=39.5GiB | 49.4% done | ETA(epoch): 1082s + [Epoch 148][20/66] loss=0.941959 avg=0.846128 VRAM=39.2GiB | 49.4% done | ETA(epoch): 893s + [Epoch 148][30/66] loss=0.975563 avg=0.863905 VRAM=39.5GiB | 49.5% done | ETA(epoch): 699s + [Epoch 148][40/66] loss=0.713499 avg=0.859424 VRAM=39.2GiB | 49.5% done | ETA(epoch): 505s + [Epoch 148][50/66] loss=0.868105 avg=0.858884 VRAM=39.5GiB | 49.6% done | ETA(epoch): 310s + [Epoch 148][60/66] loss=0.821184 avg=0.860533 VRAM=39.2GiB | 49.6% done | ETA(epoch): 116s + Train loss: 0.858972 (1278.9s) ETA: 3542min + Val loss: 0.838518 [t_0.0-0.2=1.0708 t_0.2-0.4=0.9783 t_0.4-0.6=0.8523 t_0.6-0.8=0.7141 t_0.8-1.0=0.6280] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 149/299 (50% done) --- + [Epoch 149][10/66] loss=0.781165 avg=0.860744 VRAM=39.5GiB | 49.7% done | ETA(epoch): 1083s + [Epoch 149][20/66] loss=0.918824 avg=0.845886 VRAM=39.2GiB | 49.8% done | ETA(epoch): 889s + [Epoch 149][30/66] loss=0.711374 avg=0.837584 VRAM=39.5GiB | 49.8% done | ETA(epoch): 695s + [Epoch 149][40/66] loss=0.852484 avg=0.836619 VRAM=39.2GiB | 49.9% done | ETA(epoch): 502s + [Epoch 149][50/66] loss=0.786720 avg=0.846994 VRAM=39.5GiB | 49.9% done | ETA(epoch): 309s + [Epoch 149][60/66] loss=0.844409 avg=0.853869 VRAM=39.2GiB | 50.0% done | ETA(epoch): 116s + Train loss: 0.858772 (1276.9s) ETA: 3518min + Val loss: 0.820827 [t_0.0-0.2=1.0609 t_0.2-0.4=1.0395 t_0.4-0.6=0.8653 t_0.6-0.8=0.7137 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 19.3/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 150/299 (50% done) --- + [MilestoneVis] train_0 step 9900 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9900 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9900 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9900 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9900 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9900 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9900 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9900 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9900 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9900 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/66] loss=0.795432 avg=0.817971 VRAM=39.5GiB | 50.1% done | ETA(epoch): 6845s + [Epoch 150][20/66] loss=0.897978 avg=0.839572 VRAM=39.2GiB | 50.1% done | ETA(epoch): 3255s + [Epoch 150][30/66] loss=0.974663 avg=0.840814 VRAM=39.5GiB | 50.2% done | ETA(epoch): 1929s + [Epoch 150][40/66] loss=0.912952 avg=0.839591 VRAM=39.2GiB | 50.2% done | ETA(epoch): 1171s + [Epoch 150][50/66] loss=0.856876 avg=0.840642 VRAM=39.5GiB | 50.3% done | ETA(epoch): 638s + [Epoch 150][60/66] loss=0.972051 avg=0.849711 VRAM=39.2GiB | 50.3% done | ETA(epoch): 219s + Train loss: 0.846653 (2301.8s) ETA: 3511min + Val loss: 0.848636 [t_0.0-0.2=1.0702 t_0.2-0.4=0.9498 t_0.4-0.6=0.8373 t_0.6-0.8=0.7284 t_0.8-1.0=0.6208] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 151/299 (50% done) --- + [Epoch 151][10/66] loss=0.787522 avg=0.854280 VRAM=39.5GiB | 50.4% done | ETA(epoch): 1076s + [Epoch 151][20/66] loss=0.934367 avg=0.844981 VRAM=39.2GiB | 50.4% done | ETA(epoch): 884s + [Epoch 151][30/66] loss=0.931791 avg=0.850744 VRAM=39.5GiB | 50.5% done | ETA(epoch): 691s + [Epoch 151][40/66] loss=0.866920 avg=0.849020 VRAM=39.2GiB | 50.5% done | ETA(epoch): 499s + [Epoch 151][50/66] loss=0.919611 avg=0.853500 VRAM=39.5GiB | 50.6% done | ETA(epoch): 308s + [Epoch 151][60/66] loss=0.888269 avg=0.851390 VRAM=39.2GiB | 50.6% done | ETA(epoch): 115s + Train loss: 0.854572 (1270.2s) ETA: 3487min + Val loss: 0.840115 [t_0.0-0.2=1.0635 t_0.2-0.4=0.9958 t_0.4-0.6=0.8550 t_0.6-0.8=0.6928 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 152/299 (51% done) --- + [Epoch 152][10/66] loss=0.731761 avg=0.797936 VRAM=39.5GiB | 50.7% done | ETA(epoch): 1075s + [Epoch 152][20/66] loss=0.879923 avg=0.820056 VRAM=39.2GiB | 50.8% done | ETA(epoch): 884s + [Epoch 152][30/66] loss=0.897369 avg=0.838007 VRAM=39.5GiB | 50.8% done | ETA(epoch): 692s + [Epoch 152][40/66] loss=0.854452 avg=0.846755 VRAM=39.2GiB | 50.9% done | ETA(epoch): 501s + [Epoch 152][50/66] loss=0.990917 avg=0.854000 VRAM=39.5GiB | 50.9% done | ETA(epoch): 308s + [Epoch 152][60/66] loss=0.909827 avg=0.851401 VRAM=39.2GiB | 51.0% done | ETA(epoch): 116s + Train loss: 0.850212 (1272.0s) ETA: 3462min + Val loss: 0.856187 [t_0.0-0.2=1.0541 t_0.2-0.4=0.9747 t_0.4-0.6=0.8109 t_0.6-0.8=0.7137 t_0.8-1.0=0.6176] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 153/299 (51% done) --- + [Epoch 153][10/66] loss=0.913017 avg=0.822071 VRAM=39.5GiB | 51.1% done | ETA(epoch): 1077s + [Epoch 153][20/66] loss=0.815547 avg=0.828228 VRAM=39.2GiB | 51.1% done | ETA(epoch): 884s + [Epoch 153][30/66] loss=0.950178 avg=0.850999 VRAM=39.5GiB | 51.2% done | ETA(epoch): 693s + [Epoch 153][40/66] loss=0.848248 avg=0.846784 VRAM=39.2GiB | 51.2% done | ETA(epoch): 502s + [Epoch 153][50/66] loss=0.807794 avg=0.845744 VRAM=39.5GiB | 51.3% done | ETA(epoch): 309s + [Epoch 153][60/66] loss=0.794605 avg=0.844201 VRAM=39.2GiB | 51.3% done | ETA(epoch): 116s + Train loss: 0.843066 (1272.5s) ETA: 3438min + Val loss: 0.849803 [t_0.0-0.2=1.0810 t_0.2-0.4=0.9686 t_0.4-0.6=0.8559 t_0.6-0.8=0.6964 t_0.8-1.0=0.6376] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 154/299 (51% done) --- + [Epoch 154][10/66] loss=0.827874 avg=0.866344 VRAM=39.5GiB | 51.4% done | ETA(epoch): 1077s + [Epoch 154][20/66] loss=0.791606 avg=0.855821 VRAM=39.2GiB | 51.4% done | ETA(epoch): 884s + [Epoch 154][30/66] loss=0.942414 avg=0.869978 VRAM=39.5GiB | 51.5% done | ETA(epoch): 691s + [Epoch 154][40/66] loss=0.808459 avg=0.865452 VRAM=39.2GiB | 51.5% done | ETA(epoch): 499s + [Epoch 154][50/66] loss=0.775273 avg=0.862935 VRAM=39.5GiB | 51.6% done | ETA(epoch): 307s + [Epoch 154][60/66] loss=0.907793 avg=0.862014 VRAM=39.2GiB | 51.6% done | ETA(epoch): 115s + Train loss: 0.860745 (1267.0s) ETA: 3414min + Val loss: 0.840892 [t_0.0-0.2=1.0758 t_0.2-0.4=0.9682 t_0.4-0.6=0.8458 t_0.6-0.8=0.7234 t_0.8-1.0=0.6277] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 155/299 (52% done) --- + [Epoch 155][10/66] loss=0.770897 avg=0.858984 VRAM=39.5GiB | 51.7% done | ETA(epoch): 1074s + [Epoch 155][20/66] loss=0.827655 avg=0.846841 VRAM=39.2GiB | 51.8% done | ETA(epoch): 887s + [Epoch 155][30/66] loss=0.975174 avg=0.849971 VRAM=39.5GiB | 51.8% done | ETA(epoch): 693s + [Epoch 155][40/66] loss=0.847980 avg=0.844718 VRAM=39.2GiB | 51.9% done | ETA(epoch): 501s + [Epoch 155][50/66] loss=0.869571 avg=0.847486 VRAM=39.5GiB | 51.9% done | ETA(epoch): 308s + [Epoch 155][60/66] loss=0.957043 avg=0.847118 VRAM=39.2GiB | 52.0% done | ETA(epoch): 115s + Train loss: 0.851997 (1270.0s) ETA: 3390min + Val loss: 0.849672 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9848 t_0.4-0.6=0.8173 t_0.6-0.8=0.7295 t_0.8-1.0=0.6245] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 156/299 (52% done) --- + [Epoch 156][10/66] loss=0.837303 avg=0.863522 VRAM=39.5GiB | 52.1% done | ETA(epoch): 1077s + [Epoch 156][20/66] loss=0.851123 avg=0.866257 VRAM=39.2GiB | 52.1% done | ETA(epoch): 884s + [Epoch 156][30/66] loss=0.809882 avg=0.871489 VRAM=39.5GiB | 52.2% done | ETA(epoch): 692s + [Epoch 156][40/66] loss=0.863036 avg=0.863148 VRAM=39.2GiB | 52.2% done | ETA(epoch): 499s + [Epoch 156][50/66] loss=0.866641 avg=0.861637 VRAM=39.5GiB | 52.3% done | ETA(epoch): 307s + [Epoch 156][60/66] loss=0.827381 avg=0.863680 VRAM=39.2GiB | 52.3% done | ETA(epoch): 115s + Train loss: 0.859991 (1266.9s) ETA: 3365min + Val loss: 0.850936 [t_0.0-0.2=1.0840 t_0.2-0.4=1.0033 t_0.4-0.6=0.8572 t_0.6-0.8=0.7025 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 157/299 (52% done) --- + [Epoch 157][10/66] loss=0.814381 avg=0.838574 VRAM=39.5GiB | 52.4% done | ETA(epoch): 1074s + [Epoch 157][20/66] loss=0.824438 avg=0.853737 VRAM=39.2GiB | 52.4% done | ETA(epoch): 882s + [Epoch 157][30/66] loss=0.877782 avg=0.856944 VRAM=39.5GiB | 52.5% done | ETA(epoch): 690s + [Epoch 157][40/66] loss=0.849369 avg=0.853118 VRAM=39.2GiB | 52.5% done | ETA(epoch): 498s + [Epoch 157][50/66] loss=0.883473 avg=0.850655 VRAM=39.5GiB | 52.6% done | ETA(epoch): 307s + [Epoch 157][60/66] loss=0.872387 avg=0.852664 VRAM=39.2GiB | 52.6% done | ETA(epoch): 115s + Train loss: 0.854670 (1264.6s) ETA: 3341min + Val loss: 0.891038 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9923 t_0.4-0.6=0.8200 t_0.6-0.8=0.7466 t_0.8-1.0=0.6378] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 20.1/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 158/299 (53% done) --- + [Epoch 158][10/66] loss=0.826496 avg=0.858795 VRAM=39.5GiB | 52.7% done | ETA(epoch): 1072s + [Epoch 158][20/66] loss=0.862993 avg=0.868730 VRAM=39.2GiB | 52.8% done | ETA(epoch): 882s + [Epoch 158][30/66] loss=0.842572 avg=0.855156 VRAM=39.5GiB | 52.8% done | ETA(epoch): 690s + [Epoch 158][40/66] loss=0.888890 avg=0.848973 VRAM=39.2GiB | 52.9% done | ETA(epoch): 498s + [Epoch 158][50/66] loss=0.823262 avg=0.849623 VRAM=39.5GiB | 52.9% done | ETA(epoch): 307s + [Epoch 158][60/66] loss=0.951172 avg=0.849532 VRAM=39.2GiB | 53.0% done | ETA(epoch): 115s + Train loss: 0.852737 (1264.9s) ETA: 3317min + Val loss: 0.844519 [t_0.0-0.2=1.0592 t_0.2-0.4=0.9835 t_0.4-0.6=0.8496 t_0.6-0.8=0.7392 t_0.8-1.0=0.6397] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 159/299 (53% done) --- + [Epoch 159][10/66] loss=0.888213 avg=0.829250 VRAM=39.5GiB | 53.1% done | ETA(epoch): 1072s + [Epoch 159][20/66] loss=0.834617 avg=0.850292 VRAM=39.2GiB | 53.1% done | ETA(epoch): 881s + [Epoch 159][30/66] loss=0.981624 avg=0.856305 VRAM=39.5GiB | 53.2% done | ETA(epoch): 690s + [Epoch 159][40/66] loss=0.776950 avg=0.843874 VRAM=39.2GiB | 53.2% done | ETA(epoch): 498s + [Epoch 159][50/66] loss=0.739914 avg=0.843089 VRAM=39.5GiB | 53.3% done | ETA(epoch): 307s + [Epoch 159][60/66] loss=0.906493 avg=0.851312 VRAM=39.2GiB | 53.3% done | ETA(epoch): 115s + Train loss: 0.855052 (1265.7s) ETA: 3292min + Val loss: 0.858304 [t_0.0-0.2=1.0592 t_0.2-0.4=1.0061 t_0.4-0.6=0.8684 t_0.6-0.8=0.7304 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 160/299 (53% done) --- + [Epoch 160][10/66] loss=1.000221 avg=0.936294 VRAM=39.5GiB | 53.4% done | ETA(epoch): 1071s + [Epoch 160][20/66] loss=1.003722 avg=0.926061 VRAM=39.2GiB | 53.4% done | ETA(epoch): 880s + [Epoch 160][30/66] loss=0.877569 avg=0.896763 VRAM=39.5GiB | 53.5% done | ETA(epoch): 689s + [Epoch 160][40/66] loss=0.948675 avg=0.890792 VRAM=39.2GiB | 53.5% done | ETA(epoch): 499s + [Epoch 160][50/66] loss=0.886579 avg=0.879564 VRAM=39.5GiB | 53.6% done | ETA(epoch): 307s + [Epoch 160][60/66] loss=0.876390 avg=0.875322 VRAM=39.2GiB | 53.6% done | ETA(epoch): 115s + Train loss: 0.875634 (1265.3s) ETA: 3268min + Val loss: 0.831225 [t_0.0-0.2=1.0846 t_0.2-0.4=0.9533 t_0.4-0.6=0.8644 t_0.6-0.8=0.7092 t_0.8-1.0=0.6321] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 161/299 (54% done) --- + [Epoch 161][10/66] loss=0.830312 avg=0.829420 VRAM=39.5GiB | 53.7% done | ETA(epoch): 1070s + [Epoch 161][20/66] loss=0.800918 avg=0.848974 VRAM=39.2GiB | 53.8% done | ETA(epoch): 880s + [Epoch 161][30/66] loss=0.953557 avg=0.853484 VRAM=39.5GiB | 53.8% done | ETA(epoch): 689s + [Epoch 161][40/66] loss=0.975161 avg=0.852568 VRAM=39.2GiB | 53.9% done | ETA(epoch): 497s + [Epoch 161][50/66] loss=0.733318 avg=0.852069 VRAM=39.5GiB | 53.9% done | ETA(epoch): 306s + [Epoch 161][60/66] loss=0.892780 avg=0.853287 VRAM=39.2GiB | 54.0% done | ETA(epoch): 115s + Train loss: 0.854880 (1262.8s) ETA: 3244min + Val loss: 0.869468 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9823 t_0.4-0.6=0.8393 t_0.6-0.8=0.6854 t_0.8-1.0=0.6410] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 162/299 (54% done) --- + [Epoch 162][10/66] loss=0.843191 avg=0.844675 VRAM=39.5GiB | 54.1% done | ETA(epoch): 1071s + [Epoch 162][20/66] loss=0.855283 avg=0.858921 VRAM=39.2GiB | 54.1% done | ETA(epoch): 881s + [Epoch 162][30/66] loss=0.774280 avg=0.861568 VRAM=39.5GiB | 54.2% done | ETA(epoch): 692s + [Epoch 162][40/66] loss=0.831533 avg=0.854998 VRAM=39.2GiB | 54.2% done | ETA(epoch): 499s + [Epoch 162][50/66] loss=0.778443 avg=0.854254 VRAM=39.5GiB | 54.3% done | ETA(epoch): 307s + [Epoch 162][60/66] loss=0.882970 avg=0.853945 VRAM=39.2GiB | 54.3% done | ETA(epoch): 115s + Train loss: 0.851036 (1265.4s) ETA: 3220min + Val loss: 0.841701 [t_0.0-0.2=1.0733 t_0.2-0.4=1.0271 t_0.4-0.6=0.8812 t_0.6-0.8=0.7144 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 163/299 (54% done) --- + [Epoch 163][10/66] loss=0.770764 avg=0.849550 VRAM=39.5GiB | 54.4% done | ETA(epoch): 1072s + [Epoch 163][20/66] loss=0.813293 avg=0.864752 VRAM=39.2GiB | 54.4% done | ETA(epoch): 880s + [Epoch 163][30/66] loss=0.739851 avg=0.860690 VRAM=39.5GiB | 54.5% done | ETA(epoch): 689s + [Epoch 163][40/66] loss=0.958881 avg=0.862193 VRAM=39.2GiB | 54.5% done | ETA(epoch): 497s + [Epoch 163][50/66] loss=0.861048 avg=0.859891 VRAM=39.5GiB | 54.6% done | ETA(epoch): 306s + [Epoch 163][60/66] loss=0.898587 avg=0.854827 VRAM=39.2GiB | 54.6% done | ETA(epoch): 115s + Train loss: 0.853662 (1262.9s) ETA: 3196min + Val loss: 0.850829 [t_0.0-0.2=1.0776 t_0.2-0.4=1.0031 t_0.4-0.6=0.8651 t_0.6-0.8=0.7039 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 164/299 (55% done) --- + [Epoch 164][10/66] loss=0.875167 avg=0.854044 VRAM=39.5GiB | 54.7% done | ETA(epoch): 1094s + [Epoch 164][20/66] loss=0.898477 avg=0.874782 VRAM=39.2GiB | 54.8% done | ETA(epoch): 890s + [Epoch 164][30/66] loss=0.884367 avg=0.861922 VRAM=39.5GiB | 54.8% done | ETA(epoch): 694s + [Epoch 164][40/66] loss=0.895063 avg=0.859484 VRAM=39.2GiB | 54.9% done | ETA(epoch): 500s + [Epoch 164][50/66] loss=0.894564 avg=0.860869 VRAM=39.5GiB | 54.9% done | ETA(epoch): 308s + [Epoch 164][60/66] loss=0.821264 avg=0.857610 VRAM=39.2GiB | 55.0% done | ETA(epoch): 115s + Train loss: 0.862067 (1268.1s) ETA: 3171min + Val loss: 0.857961 [t_0.0-0.2=1.0606 t_0.2-0.4=0.9679 t_0.4-0.6=0.8456 t_0.6-0.8=0.7070 t_0.8-1.0=0.6413] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 165/299 (55% done) --- + [Epoch 165][10/66] loss=0.950678 avg=0.870551 VRAM=39.5GiB | 55.1% done | ETA(epoch): 1069s + [Epoch 165][20/66] loss=0.910748 avg=0.866913 VRAM=39.2GiB | 55.1% done | ETA(epoch): 879s + [Epoch 165][30/66] loss=0.998076 avg=0.868654 VRAM=39.5GiB | 55.2% done | ETA(epoch): 688s + [Epoch 165][40/66] loss=0.769908 avg=0.865847 VRAM=39.2GiB | 55.2% done | ETA(epoch): 497s + [Epoch 165][50/66] loss=0.787863 avg=0.869615 VRAM=39.5GiB | 55.3% done | ETA(epoch): 306s + [Epoch 165][60/66] loss=0.853285 avg=0.868383 VRAM=39.2GiB | 55.3% done | ETA(epoch): 115s + Train loss: 0.860701 (1261.5s) ETA: 3147min + Val loss: 0.880729 [t_0.0-0.2=1.0652 t_0.2-0.4=1.0117 t_0.4-0.6=0.8242 t_0.6-0.8=0.7151 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 166/299 (55% done) --- + [Epoch 166][10/66] loss=0.915387 avg=0.875186 VRAM=39.5GiB | 55.4% done | ETA(epoch): 1070s + [Epoch 166][20/66] loss=0.864948 avg=0.873443 VRAM=39.2GiB | 55.4% done | ETA(epoch): 882s + [Epoch 166][30/66] loss=0.949553 avg=0.864445 VRAM=39.5GiB | 55.5% done | ETA(epoch): 689s + [Epoch 166][40/66] loss=0.769623 avg=0.852984 VRAM=39.2GiB | 55.5% done | ETA(epoch): 498s + [Epoch 166][50/66] loss=0.887924 avg=0.851032 VRAM=39.5GiB | 55.6% done | ETA(epoch): 306s + [Epoch 166][60/66] loss=0.827256 avg=0.847643 VRAM=39.2GiB | 55.6% done | ETA(epoch): 115s + Train loss: 0.844520 (1264.9s) ETA: 3123min + Val loss: 0.858674 [t_0.0-0.2=1.0638 t_0.2-0.4=0.9843 t_0.4-0.6=0.8198 t_0.6-0.8=0.7301 t_0.8-1.0=0.6343] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 167/299 (56% done) --- + [Epoch 167][10/66] loss=0.932795 avg=0.872806 VRAM=39.5GiB | 55.7% done | ETA(epoch): 1070s + [Epoch 167][20/66] loss=0.930909 avg=0.849050 VRAM=39.2GiB | 55.8% done | ETA(epoch): 880s + [Epoch 167][30/66] loss=0.668512 avg=0.841398 VRAM=39.5GiB | 55.8% done | ETA(epoch): 689s + [Epoch 167][40/66] loss=0.803618 avg=0.844467 VRAM=39.2GiB | 55.9% done | ETA(epoch): 497s + [Epoch 167][50/66] loss=0.775813 avg=0.842143 VRAM=39.5GiB | 55.9% done | ETA(epoch): 306s + [Epoch 167][60/66] loss=0.804931 avg=0.842315 VRAM=39.2GiB | 56.0% done | ETA(epoch): 115s + Train loss: 0.841557 (1262.6s) ETA: 3099min + Val loss: 0.853481 [t_0.0-0.2=1.0706 t_0.2-0.4=1.0019 t_0.4-0.6=0.8405 t_0.6-0.8=0.7177 t_0.8-1.0=0.6542] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 168/299 (56% done) --- + [Epoch 168][10/66] loss=0.759398 avg=0.829904 VRAM=39.5GiB | 56.1% done | ETA(epoch): 1071s + [Epoch 168][20/66] loss=0.895989 avg=0.847351 VRAM=39.2GiB | 56.1% done | ETA(epoch): 880s + [Epoch 168][30/66] loss=0.864878 avg=0.851553 VRAM=39.5GiB | 56.2% done | ETA(epoch): 688s + [Epoch 168][40/66] loss=0.871385 avg=0.856327 VRAM=39.2GiB | 56.2% done | ETA(epoch): 497s + [Epoch 168][50/66] loss=0.880678 avg=0.855698 VRAM=39.5GiB | 56.3% done | ETA(epoch): 306s + [Epoch 168][60/66] loss=0.847684 avg=0.856271 VRAM=39.2GiB | 56.3% done | ETA(epoch): 115s + Train loss: 0.857420 (1262.6s) ETA: 3075min + Val loss: 0.827182 [t_0.0-0.2=1.0832 t_0.2-0.4=1.0122 t_0.4-0.6=0.8183 t_0.6-0.8=0.6934 t_0.8-1.0=0.6212] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 169/299 (56% done) --- + [Epoch 169][10/66] loss=0.795973 avg=0.804838 VRAM=39.5GiB | 56.4% done | ETA(epoch): 1070s + [Epoch 169][20/66] loss=0.830432 avg=0.817457 VRAM=39.2GiB | 56.4% done | ETA(epoch): 879s + [Epoch 169][30/66] loss=0.992609 avg=0.832672 VRAM=39.5GiB | 56.5% done | ETA(epoch): 691s + [Epoch 169][40/66] loss=0.769766 avg=0.827119 VRAM=39.2GiB | 56.5% done | ETA(epoch): 499s + [Epoch 169][50/66] loss=0.740198 avg=0.830845 VRAM=39.5GiB | 56.6% done | ETA(epoch): 307s + [Epoch 169][60/66] loss=0.840963 avg=0.832805 VRAM=39.2GiB | 56.6% done | ETA(epoch): 115s + Train loss: 0.836793 (1264.9s) ETA: 3051min + Val loss: 0.848815 [t_0.0-0.2=1.0769 t_0.2-0.4=1.0506 t_0.4-0.6=0.8262 t_0.6-0.8=0.7003 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 170/299 (57% done) --- + [Epoch 170][10/66] loss=0.914953 avg=0.836496 VRAM=39.5GiB | 56.7% done | ETA(epoch): 1071s + [Epoch 170][20/66] loss=0.994193 avg=0.869483 VRAM=39.2GiB | 56.8% done | ETA(epoch): 880s + [Epoch 170][30/66] loss=0.850418 avg=0.856946 VRAM=39.5GiB | 56.8% done | ETA(epoch): 689s + [Epoch 170][40/66] loss=0.785451 avg=0.851696 VRAM=39.2GiB | 56.9% done | ETA(epoch): 497s + [Epoch 170][50/66] loss=0.874143 avg=0.858998 VRAM=39.5GiB | 56.9% done | ETA(epoch): 306s + [Epoch 170][60/66] loss=0.882024 avg=0.862378 VRAM=39.2GiB | 57.0% done | ETA(epoch): 115s + Train loss: 0.865986 (1262.4s) ETA: 3027min + Val loss: 0.835154 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9856 t_0.4-0.6=0.8214 t_0.6-0.8=0.7276 t_0.8-1.0=0.6330] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 171/299 (57% done) --- + [Epoch 171][10/66] loss=0.896962 avg=0.816356 VRAM=39.5GiB | 57.1% done | ETA(epoch): 1070s + [Epoch 171][20/66] loss=0.793972 avg=0.838759 VRAM=39.2GiB | 57.1% done | ETA(epoch): 879s + [Epoch 171][30/66] loss=0.730518 avg=0.842049 VRAM=39.5GiB | 57.2% done | ETA(epoch): 689s + [Epoch 171][40/66] loss=0.824136 avg=0.851400 VRAM=39.2GiB | 57.2% done | ETA(epoch): 497s + [Epoch 171][50/66] loss=0.832458 avg=0.847370 VRAM=39.5GiB | 57.3% done | ETA(epoch): 306s + [Epoch 171][60/66] loss=0.747221 avg=0.849878 VRAM=39.2GiB | 57.3% done | ETA(epoch): 115s + Train loss: 0.850049 (1262.0s) ETA: 3002min + Val loss: 0.841779 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9883 t_0.4-0.6=0.8310 t_0.6-0.8=0.6895 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 172/299 (57% done) --- + [Epoch 172][10/66] loss=0.830842 avg=0.853845 VRAM=39.5GiB | 57.4% done | ETA(epoch): 1082s + [Epoch 172][20/66] loss=0.863048 avg=0.850100 VRAM=39.2GiB | 57.4% done | ETA(epoch): 885s + [Epoch 172][30/66] loss=0.932007 avg=0.866581 VRAM=39.5GiB | 57.5% done | ETA(epoch): 691s + [Epoch 172][40/66] loss=0.852020 avg=0.859076 VRAM=39.2GiB | 57.5% done | ETA(epoch): 499s + [Epoch 172][50/66] loss=0.748833 avg=0.863304 VRAM=39.5GiB | 57.6% done | ETA(epoch): 307s + [Epoch 172][60/66] loss=0.804920 avg=0.856087 VRAM=39.2GiB | 57.6% done | ETA(epoch): 115s + Train loss: 0.855453 (1264.9s) ETA: 2978min + Val loss: 0.827632 [t_0.0-0.2=1.0825 t_0.2-0.4=0.9670 t_0.4-0.6=0.8288 t_0.6-0.8=0.7015 t_0.8-1.0=0.6222] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 173/299 (58% done) --- + [Epoch 173][10/66] loss=0.872334 avg=0.875783 VRAM=39.5GiB | 57.7% done | ETA(epoch): 1070s + [Epoch 173][20/66] loss=0.738133 avg=0.839821 VRAM=39.2GiB | 57.8% done | ETA(epoch): 879s + [Epoch 173][30/66] loss=0.782321 avg=0.847597 VRAM=39.5GiB | 57.8% done | ETA(epoch): 688s + [Epoch 173][40/66] loss=0.776351 avg=0.848100 VRAM=39.2GiB | 57.9% done | ETA(epoch): 497s + [Epoch 173][50/66] loss=0.721840 avg=0.839614 VRAM=39.5GiB | 57.9% done | ETA(epoch): 306s + [Epoch 173][60/66] loss=0.875785 avg=0.845098 VRAM=39.2GiB | 58.0% done | ETA(epoch): 115s + Train loss: 0.850507 (1262.1s) ETA: 2954min + Val loss: 0.879684 [t_0.0-0.2=1.0833 t_0.2-0.4=0.9799 t_0.4-0.6=0.8429 t_0.6-0.8=0.7196 t_0.8-1.0=0.6506] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 174/299 (58% done) --- + [Epoch 174][10/66] loss=0.719699 avg=0.819995 VRAM=39.5GiB | 58.1% done | ETA(epoch): 1069s + [Epoch 174][20/66] loss=0.870423 avg=0.842509 VRAM=39.2GiB | 58.1% done | ETA(epoch): 879s + [Epoch 174][30/66] loss=0.915609 avg=0.841244 VRAM=39.5GiB | 58.2% done | ETA(epoch): 688s + [Epoch 174][40/66] loss=0.904052 avg=0.847776 VRAM=39.2GiB | 58.2% done | ETA(epoch): 497s + [Epoch 174][50/66] loss=0.958875 avg=0.847686 VRAM=39.5GiB | 58.3% done | ETA(epoch): 306s + [Epoch 174][60/66] loss=0.807621 avg=0.856147 VRAM=39.2GiB | 58.3% done | ETA(epoch): 115s + Train loss: 0.852014 (1261.9s) ETA: 2930min + Val loss: 0.827983 [t_0.0-0.2=1.0505 t_0.2-0.4=0.9855 t_0.4-0.6=0.8138 t_0.6-0.8=0.7162 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 175/299 (58% done) --- + [Epoch 175][10/66] loss=0.758553 avg=0.882925 VRAM=39.5GiB | 58.4% done | ETA(epoch): 1072s + [Epoch 175][20/66] loss=0.846028 avg=0.868445 VRAM=39.2GiB | 58.4% done | ETA(epoch): 880s + [Epoch 175][30/66] loss=0.844060 avg=0.881813 VRAM=39.5GiB | 58.5% done | ETA(epoch): 688s + [Epoch 175][40/66] loss=0.705494 avg=0.867684 VRAM=39.2GiB | 58.5% done | ETA(epoch): 497s + [Epoch 175][50/66] loss=0.888674 avg=0.869428 VRAM=39.5GiB | 58.6% done | ETA(epoch): 306s + [Epoch 175][60/66] loss=0.829876 avg=0.867327 VRAM=39.2GiB | 58.6% done | ETA(epoch): 115s + Train loss: 0.865810 (1262.3s) ETA: 2906min + Val loss: 0.887811 [t_0.0-0.2=1.0509 t_0.2-0.4=1.0087 t_0.4-0.6=0.8321 t_0.6-0.8=0.7361 t_0.8-1.0=0.6397] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 176/299 (59% done) --- + [Epoch 176][10/66] loss=0.754236 avg=0.891442 VRAM=39.5GiB | 58.7% done | ETA(epoch): 1068s + [Epoch 176][20/66] loss=0.677607 avg=0.878302 VRAM=39.2GiB | 58.8% done | ETA(epoch): 878s + [Epoch 176][30/66] loss=0.829836 avg=0.880658 VRAM=39.5GiB | 58.8% done | ETA(epoch): 688s + [Epoch 176][40/66] loss=0.846649 avg=0.876123 VRAM=39.2GiB | 58.9% done | ETA(epoch): 498s + [Epoch 176][50/66] loss=0.765848 avg=0.862138 VRAM=39.5GiB | 58.9% done | ETA(epoch): 306s + [Epoch 176][60/66] loss=0.805116 avg=0.853958 VRAM=39.2GiB | 59.0% done | ETA(epoch): 115s + Train loss: 0.852720 (1262.9s) ETA: 2882min + Val loss: 0.858787 [t_0.0-0.2=1.0832 t_0.2-0.4=0.9752 t_0.4-0.6=0.8596 t_0.6-0.8=0.7086 t_0.8-1.0=0.6450] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 177/299 (59% done) --- + [Epoch 177][10/66] loss=0.827203 avg=0.844392 VRAM=39.5GiB | 59.1% done | ETA(epoch): 1070s + [Epoch 177][20/66] loss=0.884357 avg=0.846083 VRAM=39.2GiB | 59.1% done | ETA(epoch): 879s + [Epoch 177][30/66] loss=0.804952 avg=0.842961 VRAM=39.5GiB | 59.2% done | ETA(epoch): 688s + [Epoch 177][40/66] loss=0.927359 avg=0.855585 VRAM=39.2GiB | 59.2% done | ETA(epoch): 497s + [Epoch 177][50/66] loss=0.933973 avg=0.853336 VRAM=39.5GiB | 59.3% done | ETA(epoch): 306s + [Epoch 177][60/66] loss=0.724107 avg=0.850232 VRAM=39.2GiB | 59.3% done | ETA(epoch): 115s + Train loss: 0.851325 (1262.4s) ETA: 2858min + Val loss: 0.835246 [t_0.0-0.2=1.0626 t_0.2-0.4=0.9735 t_0.4-0.6=0.8591 t_0.6-0.8=0.7134 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 178/299 (59% done) --- + [Epoch 178][10/66] loss=0.844382 avg=0.853729 VRAM=39.5GiB | 59.4% done | ETA(epoch): 1071s + [Epoch 178][20/66] loss=0.923716 avg=0.855503 VRAM=39.2GiB | 59.4% done | ETA(epoch): 880s + [Epoch 178][30/66] loss=0.848426 avg=0.841533 VRAM=39.5GiB | 59.5% done | ETA(epoch): 688s + [Epoch 178][40/66] loss=0.738089 avg=0.846836 VRAM=39.2GiB | 59.5% done | ETA(epoch): 497s + [Epoch 178][50/66] loss=0.820329 avg=0.850494 VRAM=39.5GiB | 59.6% done | ETA(epoch): 306s + [Epoch 178][60/66] loss=0.855726 avg=0.844955 VRAM=39.2GiB | 59.6% done | ETA(epoch): 115s + Train loss: 0.843422 (1261.4s) ETA: 2834min + Val loss: 0.861763 [t_0.0-0.2=1.0686 t_0.2-0.4=0.9765 t_0.4-0.6=0.8428 t_0.6-0.8=0.7166 t_0.8-1.0=0.6375] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 179/299 (60% done) --- + [Epoch 179][10/66] loss=0.902330 avg=0.858907 VRAM=39.5GiB | 59.7% done | ETA(epoch): 1070s + [Epoch 179][20/66] loss=0.851935 avg=0.852976 VRAM=39.2GiB | 59.8% done | ETA(epoch): 880s + [Epoch 179][30/66] loss=0.833489 avg=0.841268 VRAM=39.5GiB | 59.8% done | ETA(epoch): 688s + [Epoch 179][40/66] loss=0.787794 avg=0.844745 VRAM=39.2GiB | 59.9% done | ETA(epoch): 497s + [Epoch 179][50/66] loss=0.750425 avg=0.849803 VRAM=39.5GiB | 59.9% done | ETA(epoch): 306s + [Epoch 179][60/66] loss=0.855958 avg=0.851264 VRAM=39.2GiB | 60.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 11879 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 11879 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 11879 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 11879 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 11879 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 11879 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 11879 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 11879 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 11879 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 11879 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + Train loss: 0.852100 (2290.3s) ETA: 2822min + Val loss: 0.869494 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9931 t_0.4-0.6=0.8475 t_0.6-0.8=0.7158 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 180/299 (60% done) --- + [Epoch 180][10/66] loss=0.923927 avg=0.838107 VRAM=39.5GiB | 60.1% done | ETA(epoch): 1069s + [Epoch 180][20/66] loss=0.876862 avg=0.838122 VRAM=39.2GiB | 60.1% done | ETA(epoch): 879s + [Epoch 180][30/66] loss=0.790465 avg=0.830805 VRAM=39.5GiB | 60.2% done | ETA(epoch): 688s + [Epoch 180][40/66] loss=0.979616 avg=0.839351 VRAM=39.2GiB | 60.2% done | ETA(epoch): 497s + [Epoch 180][50/66] loss=0.865994 avg=0.840720 VRAM=39.5GiB | 60.3% done | ETA(epoch): 306s + [Epoch 180][60/66] loss=0.896454 avg=0.844209 VRAM=39.2GiB | 60.3% done | ETA(epoch): 115s + Train loss: 0.845624 (1261.5s) ETA: 2798min + Val loss: 0.816539 [t_0.0-0.2=1.0609 t_0.2-0.4=1.0027 t_0.4-0.6=0.8379 t_0.6-0.8=0.7206 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 181/299 (60% done) --- + [Epoch 181][10/66] loss=0.881723 avg=0.853356 VRAM=39.5GiB | 60.4% done | ETA(epoch): 1071s + [Epoch 181][20/66] loss=0.889642 avg=0.843415 VRAM=39.2GiB | 60.4% done | ETA(epoch): 878s + [Epoch 181][30/66] loss=0.885734 avg=0.848334 VRAM=39.5GiB | 60.5% done | ETA(epoch): 688s + [Epoch 181][40/66] loss=0.812465 avg=0.841168 VRAM=39.2GiB | 60.5% done | ETA(epoch): 497s + [Epoch 181][50/66] loss=0.816117 avg=0.839317 VRAM=39.5GiB | 60.6% done | ETA(epoch): 306s + [Epoch 181][60/66] loss=0.928779 avg=0.839314 VRAM=39.2GiB | 60.6% done | ETA(epoch): 115s + Train loss: 0.842491 (1261.7s) ETA: 2774min + Val loss: 0.857702 [t_0.0-0.2=1.0848 t_0.2-0.4=1.0154 t_0.4-0.6=0.8510 t_0.6-0.8=0.7032 t_0.8-1.0=0.6442] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 182/299 (61% done) --- + [Epoch 182][10/66] loss=0.855289 avg=0.860390 VRAM=39.5GiB | 60.7% done | ETA(epoch): 1072s + [Epoch 182][20/66] loss=0.839885 avg=0.819658 VRAM=39.2GiB | 60.8% done | ETA(epoch): 880s + [Epoch 182][30/66] loss=0.678607 avg=0.831802 VRAM=39.5GiB | 60.8% done | ETA(epoch): 688s + [Epoch 182][40/66] loss=0.812564 avg=0.827468 VRAM=39.2GiB | 60.9% done | ETA(epoch): 497s + [Epoch 182][50/66] loss=0.841990 avg=0.832971 VRAM=39.5GiB | 60.9% done | ETA(epoch): 306s + [Epoch 182][60/66] loss=0.982373 avg=0.833433 VRAM=39.2GiB | 61.0% done | ETA(epoch): 115s + Train loss: 0.837131 (1261.6s) ETA: 2749min + Val loss: 0.842018 [t_0.0-0.2=1.0777 t_0.2-0.4=0.9807 t_0.4-0.6=0.8760 t_0.6-0.8=0.6983 t_0.8-1.0=0.6471] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 183/299 (61% done) --- + [Epoch 183][10/66] loss=0.791151 avg=0.860184 VRAM=39.5GiB | 61.1% done | ETA(epoch): 1069s + [Epoch 183][20/66] loss=0.913338 avg=0.866242 VRAM=39.2GiB | 61.1% done | ETA(epoch): 878s + [Epoch 183][30/66] loss=0.724986 avg=0.858147 VRAM=39.5GiB | 61.2% done | ETA(epoch): 687s + [Epoch 183][40/66] loss=0.925410 avg=0.856825 VRAM=39.2GiB | 61.2% done | ETA(epoch): 496s + [Epoch 183][50/66] loss=0.841708 avg=0.861427 VRAM=39.5GiB | 61.3% done | ETA(epoch): 306s + [Epoch 183][60/66] loss=0.869814 avg=0.858756 VRAM=39.2GiB | 61.3% done | ETA(epoch): 115s + Train loss: 0.859671 (1260.4s) ETA: 2725min + Val loss: 0.871851 [t_0.0-0.2=1.0650 t_0.2-0.4=0.9684 t_0.4-0.6=0.8390 t_0.6-0.8=0.7344 t_0.8-1.0=0.6483] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 184/299 (61% done) --- + [Epoch 184][10/66] loss=0.927725 avg=0.881300 VRAM=39.5GiB | 61.4% done | ETA(epoch): 1069s + [Epoch 184][20/66] loss=1.017474 avg=0.871585 VRAM=39.2GiB | 61.4% done | ETA(epoch): 878s + [Epoch 184][30/66] loss=0.957505 avg=0.869653 VRAM=39.5GiB | 61.5% done | ETA(epoch): 687s + [Epoch 184][40/66] loss=0.851397 avg=0.865622 VRAM=39.2GiB | 61.5% done | ETA(epoch): 496s + [Epoch 184][50/66] loss=0.789437 avg=0.857966 VRAM=39.5GiB | 61.6% done | ETA(epoch): 308s + [Epoch 184][60/66] loss=0.897514 avg=0.857672 VRAM=39.2GiB | 61.6% done | ETA(epoch): 115s + Train loss: 0.859005 (1266.9s) ETA: 2701min + Val loss: 0.889614 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9689 t_0.4-0.6=0.8626 t_0.6-0.8=0.7085 t_0.8-1.0=0.6347] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 185/299 (62% done) --- + [Epoch 185][10/66] loss=0.891078 avg=0.843258 VRAM=39.5GiB | 61.7% done | ETA(epoch): 1068s + [Epoch 185][20/66] loss=0.928286 avg=0.844537 VRAM=39.2GiB | 61.8% done | ETA(epoch): 879s + [Epoch 185][30/66] loss=0.743725 avg=0.853841 VRAM=39.5GiB | 61.8% done | ETA(epoch): 688s + [Epoch 185][40/66] loss=0.735822 avg=0.852483 VRAM=39.2GiB | 61.9% done | ETA(epoch): 499s + [Epoch 185][50/66] loss=0.811109 avg=0.847380 VRAM=39.5GiB | 61.9% done | ETA(epoch): 307s + [Epoch 185][60/66] loss=0.750479 avg=0.844433 VRAM=39.2GiB | 62.0% done | ETA(epoch): 115s + Train loss: 0.846711 (1263.5s) ETA: 2677min + Val loss: 0.820857 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9894 t_0.4-0.6=0.8203 t_0.6-0.8=0.7029 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 186/299 (62% done) --- + [Epoch 186][10/66] loss=0.809680 avg=0.845064 VRAM=39.5GiB | 62.1% done | ETA(epoch): 1068s + [Epoch 186][20/66] loss=0.670712 avg=0.832234 VRAM=39.2GiB | 62.1% done | ETA(epoch): 878s + [Epoch 186][30/66] loss=0.976287 avg=0.833388 VRAM=39.5GiB | 62.2% done | ETA(epoch): 687s + [Epoch 186][40/66] loss=0.784981 avg=0.834239 VRAM=39.2GiB | 62.2% done | ETA(epoch): 496s + [Epoch 186][50/66] loss=0.862185 avg=0.844018 VRAM=39.5GiB | 62.3% done | ETA(epoch): 305s + [Epoch 186][60/66] loss=0.784192 avg=0.849817 VRAM=39.2GiB | 62.3% done | ETA(epoch): 114s + Train loss: 0.849722 (1259.7s) ETA: 2653min + Val loss: 0.823418 [t_0.0-0.2=1.0511 t_0.2-0.4=1.0144 t_0.4-0.6=0.8667 t_0.6-0.8=0.7215 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 187/299 (62% done) --- + [Epoch 187][10/66] loss=0.922751 avg=0.849631 VRAM=39.5GiB | 62.4% done | ETA(epoch): 1070s + [Epoch 187][20/66] loss=0.867669 avg=0.847883 VRAM=39.2GiB | 62.4% done | ETA(epoch): 878s + [Epoch 187][30/66] loss=0.896101 avg=0.844485 VRAM=39.5GiB | 62.5% done | ETA(epoch): 687s + [Epoch 187][40/66] loss=0.860509 avg=0.857009 VRAM=39.2GiB | 62.5% done | ETA(epoch): 496s + [Epoch 187][50/66] loss=0.966179 avg=0.858177 VRAM=39.5GiB | 62.6% done | ETA(epoch): 305s + [Epoch 187][60/66] loss=0.894535 avg=0.864343 VRAM=39.2GiB | 62.6% done | ETA(epoch): 114s + Train loss: 0.859819 (1259.3s) ETA: 2629min + Val loss: 0.854380 [t_0.0-0.2=1.0766 t_0.2-0.4=0.9976 t_0.4-0.6=0.8455 t_0.6-0.8=0.6908 t_0.8-1.0=0.6331] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 188/299 (63% done) --- + [Epoch 188][10/66] loss=0.755566 avg=0.844770 VRAM=39.5GiB | 62.7% done | ETA(epoch): 1070s + [Epoch 188][20/66] loss=0.898976 avg=0.866087 VRAM=39.2GiB | 62.8% done | ETA(epoch): 888s + [Epoch 188][30/66] loss=0.733193 avg=0.857959 VRAM=39.5GiB | 62.8% done | ETA(epoch): 692s + [Epoch 188][40/66] loss=0.859149 avg=0.857439 VRAM=39.2GiB | 62.9% done | ETA(epoch): 499s + [Epoch 188][50/66] loss=0.940875 avg=0.855721 VRAM=39.5GiB | 62.9% done | ETA(epoch): 307s + [Epoch 188][60/66] loss=0.870146 avg=0.857426 VRAM=39.2GiB | 63.0% done | ETA(epoch): 115s + Train loss: 0.852560 (1263.0s) ETA: 2605min + Val loss: 0.839255 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0039 t_0.4-0.6=0.8067 t_0.6-0.8=0.7096 t_0.8-1.0=0.6443] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 189/299 (63% done) --- + [Epoch 189][10/66] loss=0.735725 avg=0.867133 VRAM=39.5GiB | 63.1% done | ETA(epoch): 1067s + [Epoch 189][20/66] loss=0.839955 avg=0.856191 VRAM=39.2GiB | 63.1% done | ETA(epoch): 877s + [Epoch 189][30/66] loss=0.769784 avg=0.846257 VRAM=39.5GiB | 63.2% done | ETA(epoch): 686s + [Epoch 189][40/66] loss=0.824514 avg=0.854148 VRAM=39.2GiB | 63.2% done | ETA(epoch): 496s + [Epoch 189][50/66] loss=0.839699 avg=0.855096 VRAM=39.5GiB | 63.3% done | ETA(epoch): 305s + [Epoch 189][60/66] loss=0.781741 avg=0.852330 VRAM=39.2GiB | 63.3% done | ETA(epoch): 114s + Train loss: 0.848444 (1258.7s) ETA: 2581min + Val loss: 0.865096 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9716 t_0.4-0.6=0.8121 t_0.6-0.8=0.7221 t_0.8-1.0=0.6427] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 190/299 (63% done) --- + [Epoch 190][10/66] loss=0.873690 avg=0.858025 VRAM=39.5GiB | 63.4% done | ETA(epoch): 1068s + [Epoch 190][20/66] loss=0.823321 avg=0.826023 VRAM=39.2GiB | 63.4% done | ETA(epoch): 877s + [Epoch 190][30/66] loss=0.956844 avg=0.843920 VRAM=39.5GiB | 63.5% done | ETA(epoch): 686s + [Epoch 190][40/66] loss=0.795111 avg=0.838462 VRAM=39.2GiB | 63.5% done | ETA(epoch): 496s + [Epoch 190][50/66] loss=0.910892 avg=0.839764 VRAM=39.5GiB | 63.6% done | ETA(epoch): 305s + [Epoch 190][60/66] loss=0.835960 avg=0.835643 VRAM=39.2GiB | 63.6% done | ETA(epoch): 115s + Train loss: 0.834942 (1261.7s) ETA: 2557min + Val loss: 0.817254 [t_0.0-0.2=1.0811 t_0.2-0.4=0.9538 t_0.4-0.6=0.8558 t_0.6-0.8=0.6966 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 191/299 (64% done) --- + [Epoch 191][10/66] loss=0.665678 avg=0.805726 VRAM=39.5GiB | 63.7% done | ETA(epoch): 1066s + [Epoch 191][20/66] loss=0.875913 avg=0.831318 VRAM=39.2GiB | 63.8% done | ETA(epoch): 876s + [Epoch 191][30/66] loss=0.935764 avg=0.845112 VRAM=39.5GiB | 63.8% done | ETA(epoch): 685s + [Epoch 191][40/66] loss=0.914675 avg=0.863970 VRAM=39.2GiB | 63.9% done | ETA(epoch): 495s + [Epoch 191][50/66] loss=0.884020 avg=0.868323 VRAM=39.5GiB | 63.9% done | ETA(epoch): 305s + [Epoch 191][60/66] loss=0.775998 avg=0.864924 VRAM=39.2GiB | 64.0% done | ETA(epoch): 114s + Train loss: 0.859848 (1256.9s) ETA: 2533min + Val loss: 0.814315 [t_0.0-0.2=1.0836 t_0.2-0.4=0.9734 t_0.4-0.6=0.8062 t_0.6-0.8=0.7240 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0191 + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 192/299 (64% done) --- + [Epoch 192][10/66] loss=0.753747 avg=0.811179 VRAM=39.5GiB | 64.1% done | ETA(epoch): 1067s + [Epoch 192][20/66] loss=0.793650 avg=0.838016 VRAM=39.2GiB | 64.1% done | ETA(epoch): 877s + [Epoch 192][30/66] loss=0.913564 avg=0.827932 VRAM=39.5GiB | 64.2% done | ETA(epoch): 686s + [Epoch 192][40/66] loss=0.910613 avg=0.829937 VRAM=39.2GiB | 64.2% done | ETA(epoch): 495s + [Epoch 192][50/66] loss=0.902699 avg=0.830428 VRAM=39.5GiB | 64.3% done | ETA(epoch): 305s + [Epoch 192][60/66] loss=0.885117 avg=0.828360 VRAM=39.2GiB | 64.3% done | ETA(epoch): 114s + Train loss: 0.830976 (1258.0s) ETA: 2509min + Val loss: 0.873615 [t_0.0-0.2=1.0561 t_0.2-0.4=0.9972 t_0.4-0.6=0.8330 t_0.6-0.8=0.7265 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 193/299 (64% done) --- + [Epoch 193][10/66] loss=0.879177 avg=0.858466 VRAM=39.5GiB | 64.4% done | ETA(epoch): 1067s + [Epoch 193][20/66] loss=0.865668 avg=0.853789 VRAM=39.2GiB | 64.4% done | ETA(epoch): 876s + [Epoch 193][30/66] loss=0.840093 avg=0.862627 VRAM=39.5GiB | 64.5% done | ETA(epoch): 686s + [Epoch 193][40/66] loss=0.639634 avg=0.851941 VRAM=39.2GiB | 64.5% done | ETA(epoch): 498s + [Epoch 193][50/66] loss=0.859699 avg=0.858057 VRAM=39.5GiB | 64.6% done | ETA(epoch): 306s + [Epoch 193][60/66] loss=0.927389 avg=0.854563 VRAM=39.2GiB | 64.6% done | ETA(epoch): 115s + Train loss: 0.852756 (1262.3s) ETA: 2485min + Val loss: 0.878629 [t_0.0-0.2=1.0719 t_0.2-0.4=0.9759 t_0.4-0.6=0.8515 t_0.6-0.8=0.7239 t_0.8-1.0=0.6143] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 194/299 (65% done) --- + [Epoch 194][10/66] loss=0.869709 avg=0.851083 VRAM=39.5GiB | 64.7% done | ETA(epoch): 1066s + [Epoch 194][20/66] loss=0.990358 avg=0.863440 VRAM=39.2GiB | 64.8% done | ETA(epoch): 876s + [Epoch 194][30/66] loss=0.742411 avg=0.854447 VRAM=39.5GiB | 64.8% done | ETA(epoch): 689s + [Epoch 194][40/66] loss=0.888695 avg=0.851870 VRAM=39.2GiB | 64.9% done | ETA(epoch): 497s + [Epoch 194][50/66] loss=0.656643 avg=0.847243 VRAM=39.5GiB | 64.9% done | ETA(epoch): 306s + [Epoch 194][60/66] loss=0.783241 avg=0.843736 VRAM=39.2GiB | 65.0% done | ETA(epoch): 115s + Train loss: 0.841720 (1260.7s) ETA: 2462min + Val loss: 0.858960 [t_0.0-0.2=1.0562 t_0.2-0.4=1.0125 t_0.4-0.6=0.8322 t_0.6-0.8=0.7214 t_0.8-1.0=0.6342] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 195/299 (65% done) --- + [Epoch 195][10/66] loss=0.975110 avg=0.851144 VRAM=39.5GiB | 65.1% done | ETA(epoch): 1066s + [Epoch 195][20/66] loss=0.880872 avg=0.851663 VRAM=39.2GiB | 65.1% done | ETA(epoch): 882s + [Epoch 195][30/66] loss=0.829937 avg=0.854790 VRAM=39.5GiB | 65.2% done | ETA(epoch): 689s + [Epoch 195][40/66] loss=0.945578 avg=0.860274 VRAM=39.2GiB | 65.2% done | ETA(epoch): 497s + [Epoch 195][50/66] loss=0.864602 avg=0.861148 VRAM=39.5GiB | 65.3% done | ETA(epoch): 306s + [Epoch 195][60/66] loss=0.907761 avg=0.857627 VRAM=39.2GiB | 65.3% done | ETA(epoch): 115s + Train loss: 0.854516 (1260.2s) ETA: 2438min + Val loss: 0.858159 [t_0.0-0.2=1.0782 t_0.2-0.4=0.9498 t_0.4-0.6=0.8628 t_0.6-0.8=0.7306 t_0.8-1.0=0.6259] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 196/299 (65% done) --- + [Epoch 196][10/66] loss=0.803434 avg=0.853165 VRAM=39.5GiB | 65.4% done | ETA(epoch): 1065s + [Epoch 196][20/66] loss=0.819257 avg=0.866264 VRAM=39.2GiB | 65.4% done | ETA(epoch): 881s + [Epoch 196][30/66] loss=0.917439 avg=0.865265 VRAM=39.5GiB | 65.5% done | ETA(epoch): 688s + [Epoch 196][40/66] loss=0.952380 avg=0.862760 VRAM=39.2GiB | 65.5% done | ETA(epoch): 497s + [Epoch 196][50/66] loss=0.918159 avg=0.855564 VRAM=39.5GiB | 65.6% done | ETA(epoch): 306s + [Epoch 196][60/66] loss=0.877516 avg=0.852175 VRAM=39.2GiB | 65.6% done | ETA(epoch): 115s + Train loss: 0.849441 (1260.0s) ETA: 2414min + Val loss: 0.814405 [t_0.0-0.2=1.0618 t_0.2-0.4=0.9742 t_0.4-0.6=0.8343 t_0.6-0.8=0.7088 t_0.8-1.0=0.6252] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 197/299 (66% done) --- + [Epoch 197][10/66] loss=0.734558 avg=0.869661 VRAM=39.5GiB | 65.7% done | ETA(epoch): 1067s + [Epoch 197][20/66] loss=0.886204 avg=0.846159 VRAM=39.2GiB | 65.8% done | ETA(epoch): 877s + [Epoch 197][30/66] loss=0.827699 avg=0.854932 VRAM=39.5GiB | 65.8% done | ETA(epoch): 686s + [Epoch 197][40/66] loss=0.647048 avg=0.843974 VRAM=39.2GiB | 65.9% done | ETA(epoch): 496s + [Epoch 197][50/66] loss=0.830446 avg=0.835626 VRAM=39.5GiB | 65.9% done | ETA(epoch): 305s + [Epoch 197][60/66] loss=0.893328 avg=0.835576 VRAM=39.2GiB | 66.0% done | ETA(epoch): 114s + Train loss: 0.838982 (1257.9s) ETA: 2390min + Val loss: 0.862004 [t_0.0-0.2=1.0681 t_0.2-0.4=1.0042 t_0.4-0.6=0.8722 t_0.6-0.8=0.7164 t_0.8-1.0=0.6455] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 198/299 (66% done) --- + [Epoch 198][10/66] loss=0.708967 avg=0.806990 VRAM=39.5GiB | 66.1% done | ETA(epoch): 1066s + [Epoch 198][20/66] loss=0.869938 avg=0.831709 VRAM=39.2GiB | 66.1% done | ETA(epoch): 876s + [Epoch 198][30/66] loss=0.756968 avg=0.840270 VRAM=39.5GiB | 66.2% done | ETA(epoch): 686s + [Epoch 198][40/66] loss=0.925957 avg=0.838325 VRAM=39.2GiB | 66.2% done | ETA(epoch): 495s + [Epoch 198][50/66] loss=0.786365 avg=0.842787 VRAM=39.5GiB | 66.3% done | ETA(epoch): 305s + [Epoch 198][60/66] loss=0.858201 avg=0.851280 VRAM=39.2GiB | 66.3% done | ETA(epoch): 114s + Train loss: 0.851322 (1257.2s) ETA: 2366min + Val loss: 0.864477 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9684 t_0.4-0.6=0.8560 t_0.6-0.8=0.7149 t_0.8-1.0=0.6287] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 199/299 (66% done) --- + [Epoch 199][10/66] loss=0.812051 avg=0.856950 VRAM=39.5GiB | 66.4% done | ETA(epoch): 1067s + [Epoch 199][20/66] loss=1.001043 avg=0.858688 VRAM=39.2GiB | 66.4% done | ETA(epoch): 877s + [Epoch 199][30/66] loss=0.901640 avg=0.866163 VRAM=39.5GiB | 66.5% done | ETA(epoch): 686s + [Epoch 199][40/66] loss=0.790244 avg=0.860068 VRAM=39.2GiB | 66.5% done | ETA(epoch): 496s + [Epoch 199][50/66] loss=0.884625 avg=0.860891 VRAM=39.5GiB | 66.6% done | ETA(epoch): 305s + [Epoch 199][60/66] loss=0.833154 avg=0.854738 VRAM=39.2GiB | 66.6% done | ETA(epoch): 114s + Train loss: 0.855644 (1257.6s) ETA: 2342min + Val loss: 0.847359 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9865 t_0.4-0.6=0.8647 t_0.6-0.8=0.7197 t_0.8-1.0=0.6283] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 200/299 (67% done) --- + [Epoch 200][10/66] loss=0.909520 avg=0.848879 VRAM=39.5GiB | 66.7% done | ETA(epoch): 1066s + [Epoch 200][20/66] loss=0.772132 avg=0.864177 VRAM=39.2GiB | 66.8% done | ETA(epoch): 877s + [Epoch 200][30/66] loss=0.732692 avg=0.851367 VRAM=39.5GiB | 66.8% done | ETA(epoch): 686s + [Epoch 200][40/66] loss=0.854042 avg=0.847725 VRAM=39.2GiB | 66.9% done | ETA(epoch): 495s + [Epoch 200][50/66] loss=0.743735 avg=0.839731 VRAM=39.5GiB | 66.9% done | ETA(epoch): 305s + [Epoch 200][60/66] loss=0.917469 avg=0.841511 VRAM=39.2GiB | 67.0% done | ETA(epoch): 114s + Train loss: 0.841914 (1258.9s) ETA: 2318min + Val loss: 0.878988 [t_0.0-0.2=1.0846 t_0.2-0.4=0.9773 t_0.4-0.6=0.8415 t_0.6-0.8=0.6983 t_0.8-1.0=0.6515] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 201/299 (67% done) --- + [Epoch 201][10/66] loss=0.916167 avg=0.872203 VRAM=39.5GiB | 67.1% done | ETA(epoch): 1064s + [Epoch 201][20/66] loss=0.928233 avg=0.869764 VRAM=39.2GiB | 67.1% done | ETA(epoch): 875s + [Epoch 201][30/66] loss=0.835783 avg=0.872112 VRAM=39.5GiB | 67.2% done | ETA(epoch): 685s + [Epoch 201][40/66] loss=0.892389 avg=0.858186 VRAM=39.2GiB | 67.2% done | ETA(epoch): 494s + [Epoch 201][50/66] loss=0.877496 avg=0.859992 VRAM=39.5GiB | 67.3% done | ETA(epoch): 304s + [Epoch 201][60/66] loss=0.925272 avg=0.865785 VRAM=39.2GiB | 67.3% done | ETA(epoch): 114s + Train loss: 0.863544 (1256.0s) ETA: 2294min + Val loss: 0.833627 [t_0.0-0.2=1.0531 t_0.2-0.4=0.9953 t_0.4-0.6=0.8643 t_0.6-0.8=0.7101 t_0.8-1.0=0.6376] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 202/299 (67% done) --- + [Epoch 202][10/66] loss=0.899252 avg=0.833168 VRAM=39.5GiB | 67.4% done | ETA(epoch): 1066s + [Epoch 202][20/66] loss=0.718526 avg=0.810472 VRAM=39.2GiB | 67.4% done | ETA(epoch): 875s + [Epoch 202][30/66] loss=0.789299 avg=0.814931 VRAM=39.5GiB | 67.5% done | ETA(epoch): 690s + [Epoch 202][40/66] loss=0.890182 avg=0.818767 VRAM=39.2GiB | 67.5% done | ETA(epoch): 498s + [Epoch 202][50/66] loss=0.832246 avg=0.827423 VRAM=39.5GiB | 67.6% done | ETA(epoch): 306s + [Epoch 202][60/66] loss=0.950190 avg=0.837107 VRAM=39.2GiB | 67.6% done | ETA(epoch): 115s + Train loss: 0.833451 (1260.8s) ETA: 2271min + Val loss: 0.834958 [t_0.0-0.2=1.0913 t_0.2-0.4=0.9813 t_0.4-0.6=0.8440 t_0.6-0.8=0.6820 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 203/299 (68% done) --- + [Epoch 203][10/66] loss=0.714253 avg=0.837026 VRAM=39.5GiB | 67.7% done | ETA(epoch): 1065s + [Epoch 203][20/66] loss=0.847831 avg=0.857417 VRAM=39.2GiB | 67.8% done | ETA(epoch): 875s + [Epoch 203][30/66] loss=0.877310 avg=0.848804 VRAM=39.5GiB | 67.8% done | ETA(epoch): 686s + [Epoch 203][40/66] loss=0.861677 avg=0.855830 VRAM=39.2GiB | 67.9% done | ETA(epoch): 495s + [Epoch 203][50/66] loss=0.880193 avg=0.856820 VRAM=39.5GiB | 67.9% done | ETA(epoch): 305s + [Epoch 203][60/66] loss=0.864387 avg=0.851527 VRAM=39.2GiB | 68.0% done | ETA(epoch): 114s + Train loss: 0.851636 (1257.0s) ETA: 2247min + Val loss: 0.855012 [t_0.0-0.2=1.0763 t_0.2-0.4=0.9545 t_0.4-0.6=0.8632 t_0.6-0.8=0.7276 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 204/299 (68% done) --- + [Epoch 204][10/66] loss=0.913338 avg=0.875413 VRAM=39.5GiB | 68.1% done | ETA(epoch): 1067s + [Epoch 204][20/66] loss=0.862715 avg=0.857086 VRAM=39.2GiB | 68.1% done | ETA(epoch): 881s + [Epoch 204][30/66] loss=0.910361 avg=0.859728 VRAM=39.5GiB | 68.2% done | ETA(epoch): 688s + [Epoch 204][40/66] loss=0.932456 avg=0.863973 VRAM=39.2GiB | 68.2% done | ETA(epoch): 497s + [Epoch 204][50/66] loss=0.971174 avg=0.864835 VRAM=39.5GiB | 68.3% done | ETA(epoch): 305s + [Epoch 204][60/66] loss=0.660131 avg=0.855064 VRAM=39.2GiB | 68.3% done | ETA(epoch): 114s + Train loss: 0.855851 (1259.2s) ETA: 2223min + Val loss: 0.853227 [t_0.0-0.2=1.0765 t_0.2-0.4=1.0136 t_0.4-0.6=0.8193 t_0.6-0.8=0.7121 t_0.8-1.0=0.6264] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 205/299 (68% done) --- + [Epoch 205][10/66] loss=0.944216 avg=0.872917 VRAM=39.5GiB | 68.4% done | ETA(epoch): 1094s + [Epoch 205][20/66] loss=1.016680 avg=0.872296 VRAM=39.2GiB | 68.4% done | ETA(epoch): 887s + [Epoch 205][30/66] loss=0.839080 avg=0.864130 VRAM=39.5GiB | 68.5% done | ETA(epoch): 691s + [Epoch 205][40/66] loss=0.879409 avg=0.866621 VRAM=39.2GiB | 68.5% done | ETA(epoch): 498s + [Epoch 205][50/66] loss=0.885652 avg=0.852816 VRAM=39.5GiB | 68.6% done | ETA(epoch): 306s + [Epoch 205][60/66] loss=0.905549 avg=0.851987 VRAM=39.2GiB | 68.6% done | ETA(epoch): 115s + Train loss: 0.855609 (1261.9s) ETA: 2199min + Val loss: 0.839153 [t_0.0-0.2=1.0798 t_0.2-0.4=0.9707 t_0.4-0.6=0.8779 t_0.6-0.8=0.7055 t_0.8-1.0=0.6296] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 206/299 (69% done) --- + [Epoch 206][10/66] loss=0.968214 avg=0.893881 VRAM=39.5GiB | 68.7% done | ETA(epoch): 1065s + [Epoch 206][20/66] loss=0.873229 avg=0.881367 VRAM=39.2GiB | 68.8% done | ETA(epoch): 876s + [Epoch 206][30/66] loss=0.946456 avg=0.872704 VRAM=39.5GiB | 68.8% done | ETA(epoch): 685s + [Epoch 206][40/66] loss=0.815877 avg=0.859175 VRAM=39.2GiB | 68.9% done | ETA(epoch): 495s + [Epoch 206][50/66] loss=0.871763 avg=0.869567 VRAM=39.5GiB | 68.9% done | ETA(epoch): 305s + [Epoch 206][60/66] loss=0.784482 avg=0.867814 VRAM=39.2GiB | 69.0% done | ETA(epoch): 114s + Train loss: 0.868702 (1256.5s) ETA: 2175min + Val loss: 0.860677 [t_0.0-0.2=1.0828 t_0.2-0.4=0.9963 t_0.4-0.6=0.8049 t_0.6-0.8=0.7198 t_0.8-1.0=0.6410] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 207/299 (69% done) --- + [Epoch 207][10/66] loss=0.716304 avg=0.830144 VRAM=39.5GiB | 69.1% done | ETA(epoch): 1066s + [Epoch 207][20/66] loss=0.834145 avg=0.842557 VRAM=39.2GiB | 69.1% done | ETA(epoch): 876s + [Epoch 207][30/66] loss=0.722833 avg=0.851216 VRAM=39.5GiB | 69.2% done | ETA(epoch): 685s + [Epoch 207][40/66] loss=0.830023 avg=0.848051 VRAM=39.2GiB | 69.2% done | ETA(epoch): 495s + [Epoch 207][50/66] loss=0.765361 avg=0.846150 VRAM=39.5GiB | 69.3% done | ETA(epoch): 305s + [Epoch 207][60/66] loss=0.941115 avg=0.843326 VRAM=39.2GiB | 69.3% done | ETA(epoch): 114s + Train loss: 0.844007 (1258.7s) ETA: 2152min + Val loss: 0.825389 [t_0.0-0.2=1.0610 t_0.2-0.4=0.9856 t_0.4-0.6=0.8168 t_0.6-0.8=0.7298 t_0.8-1.0=0.6253] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 208/299 (69% done) --- + [Epoch 208][10/66] loss=0.785501 avg=0.842160 VRAM=39.5GiB | 69.4% done | ETA(epoch): 1066s + [Epoch 208][20/66] loss=0.784403 avg=0.834793 VRAM=39.2GiB | 69.4% done | ETA(epoch): 876s + [Epoch 208][30/66] loss=0.851647 avg=0.841000 VRAM=39.5GiB | 69.5% done | ETA(epoch): 685s + [Epoch 208][40/66] loss=0.887009 avg=0.839787 VRAM=39.2GiB | 69.5% done | ETA(epoch): 495s + [Epoch 208][50/66] loss=0.785724 avg=0.840519 VRAM=39.5GiB | 69.6% done | ETA(epoch): 305s + [Epoch 208][60/66] loss=0.788414 avg=0.833818 VRAM=39.2GiB | 69.6% done | ETA(epoch): 114s + Train loss: 0.833297 (1256.5s) ETA: 2128min + Val loss: 0.819293 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9821 t_0.4-0.6=0.8119 t_0.6-0.8=0.7020 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 209/299 (70% done) --- + [Epoch 209][10/66] loss=0.937101 avg=0.869991 VRAM=39.5GiB | 69.7% done | ETA(epoch): 1064s + [Epoch 209][20/66] loss=0.821377 avg=0.854451 VRAM=39.2GiB | 69.8% done | ETA(epoch): 874s + [Epoch 209][30/66] loss=0.938559 avg=0.849801 VRAM=39.5GiB | 69.8% done | ETA(epoch): 684s + [Epoch 209][40/66] loss=0.930481 avg=0.847045 VRAM=39.2GiB | 69.9% done | ETA(epoch): 496s + [Epoch 209][50/66] loss=0.959496 avg=0.845521 VRAM=39.5GiB | 69.9% done | ETA(epoch): 305s + [Epoch 209][60/66] loss=0.967202 avg=0.852016 VRAM=39.2GiB | 70.0% done | ETA(epoch): 114s + [MilestoneVis] train_0 step 13859 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 13859 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 13859 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 13859 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 13859 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 13859 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 13859 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 13859 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 13859 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 13859 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + Train loss: 0.856688 (2283.1s) ETA: 2111min + Val loss: 0.836097 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9956 t_0.4-0.6=0.8426 t_0.6-0.8=0.7048 t_0.8-1.0=0.6348] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 21.0/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 210/299 (70% done) --- + [Epoch 210][10/66] loss=0.896861 avg=0.852034 VRAM=39.5GiB | 70.1% done | ETA(epoch): 1065s + [Epoch 210][20/66] loss=0.848435 avg=0.859388 VRAM=39.2GiB | 70.1% done | ETA(epoch): 875s + [Epoch 210][30/66] loss=0.687279 avg=0.849774 VRAM=39.5GiB | 70.2% done | ETA(epoch): 685s + [Epoch 210][40/66] loss=0.761455 avg=0.851406 VRAM=39.2GiB | 70.2% done | ETA(epoch): 494s + [Epoch 210][50/66] loss=0.803680 avg=0.844391 VRAM=39.5GiB | 70.3% done | ETA(epoch): 304s + [Epoch 210][60/66] loss=0.793433 avg=0.838667 VRAM=39.2GiB | 70.3% done | ETA(epoch): 114s + Train loss: 0.838893 (1255.8s) ETA: 2088min + Val loss: 0.858861 [t_0.0-0.2=1.0746 t_0.2-0.4=0.9881 t_0.4-0.6=0.8501 t_0.6-0.8=0.7111 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 211/299 (70% done) --- + [Epoch 211][10/66] loss=0.827239 avg=0.854493 VRAM=39.5GiB | 70.4% done | ETA(epoch): 1063s + [Epoch 211][20/66] loss=0.743094 avg=0.845721 VRAM=39.2GiB | 70.4% done | ETA(epoch): 874s + [Epoch 211][30/66] loss=0.942671 avg=0.856728 VRAM=39.5GiB | 70.5% done | ETA(epoch): 684s + [Epoch 211][40/66] loss=0.761217 avg=0.854057 VRAM=39.2GiB | 70.5% done | ETA(epoch): 496s + [Epoch 211][50/66] loss=0.866291 avg=0.850898 VRAM=39.5GiB | 70.6% done | ETA(epoch): 305s + [Epoch 211][60/66] loss=0.688317 avg=0.846504 VRAM=39.2GiB | 70.6% done | ETA(epoch): 114s + Train loss: 0.841458 (1258.0s) ETA: 2064min + Val loss: 0.822744 [t_0.0-0.2=1.0513 t_0.2-0.4=0.9718 t_0.4-0.6=0.8471 t_0.6-0.8=0.7005 t_0.8-1.0=0.6282] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 212/299 (71% done) --- + [Epoch 212][10/66] loss=0.836938 avg=0.820007 VRAM=39.5GiB | 70.7% done | ETA(epoch): 1065s + [Epoch 212][20/66] loss=1.040851 avg=0.848653 VRAM=39.2GiB | 70.8% done | ETA(epoch): 876s + [Epoch 212][30/66] loss=0.846731 avg=0.848370 VRAM=39.5GiB | 70.8% done | ETA(epoch): 688s + [Epoch 212][40/66] loss=0.821371 avg=0.847553 VRAM=39.2GiB | 70.9% done | ETA(epoch): 496s + [Epoch 212][50/66] loss=0.874884 avg=0.850118 VRAM=39.5GiB | 70.9% done | ETA(epoch): 305s + [Epoch 212][60/66] loss=0.862926 avg=0.853810 VRAM=39.2GiB | 71.0% done | ETA(epoch): 114s + Train loss: 0.855169 (1258.3s) ETA: 2040min + Val loss: 0.836700 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0023 t_0.4-0.6=0.8310 t_0.6-0.8=0.7149 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 213/299 (71% done) --- + [Epoch 213][10/66] loss=0.888197 avg=0.911974 VRAM=39.5GiB | 71.1% done | ETA(epoch): 1067s + [Epoch 213][20/66] loss=0.863691 avg=0.885428 VRAM=39.2GiB | 71.1% done | ETA(epoch): 876s + [Epoch 213][30/66] loss=0.786902 avg=0.869394 VRAM=39.5GiB | 71.2% done | ETA(epoch): 685s + [Epoch 213][40/66] loss=0.849740 avg=0.863589 VRAM=39.2GiB | 71.2% done | ETA(epoch): 495s + [Epoch 213][50/66] loss=0.845847 avg=0.858825 VRAM=39.5GiB | 71.3% done | ETA(epoch): 304s + [Epoch 213][60/66] loss=0.835827 avg=0.855578 VRAM=39.2GiB | 71.3% done | ETA(epoch): 114s + Train loss: 0.852392 (1255.8s) ETA: 2016min + Val loss: 0.825560 [t_0.0-0.2=1.0555 t_0.2-0.4=1.0316 t_0.4-0.6=0.8535 t_0.6-0.8=0.7021 t_0.8-1.0=0.6297] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 214/299 (71% done) --- + [Epoch 214][10/66] loss=0.841334 avg=0.829666 VRAM=39.5GiB | 71.4% done | ETA(epoch): 1083s + [Epoch 214][20/66] loss=0.858654 avg=0.852030 VRAM=39.2GiB | 71.4% done | ETA(epoch): 882s + [Epoch 214][30/66] loss=0.772009 avg=0.832744 VRAM=39.5GiB | 71.5% done | ETA(epoch): 689s + [Epoch 214][40/66] loss=0.999480 avg=0.839973 VRAM=39.2GiB | 71.5% done | ETA(epoch): 497s + [Epoch 214][50/66] loss=0.812938 avg=0.843444 VRAM=39.5GiB | 71.6% done | ETA(epoch): 305s + [Epoch 214][60/66] loss=0.952872 avg=0.846167 VRAM=39.2GiB | 71.6% done | ETA(epoch): 114s + Train loss: 0.845688 (1259.1s) ETA: 1992min + Val loss: 0.852506 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9668 t_0.4-0.6=0.8608 t_0.6-0.8=0.7254 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 215/299 (72% done) --- + [Epoch 215][10/66] loss=0.868600 avg=0.849101 VRAM=39.5GiB | 71.7% done | ETA(epoch): 1077s + [Epoch 215][20/66] loss=0.800177 avg=0.847596 VRAM=39.2GiB | 71.8% done | ETA(epoch): 880s + [Epoch 215][30/66] loss=0.784282 avg=0.848926 VRAM=39.5GiB | 71.8% done | ETA(epoch): 688s + [Epoch 215][40/66] loss=0.777936 avg=0.842857 VRAM=39.2GiB | 71.9% done | ETA(epoch): 496s + [Epoch 215][50/66] loss=0.866795 avg=0.846032 VRAM=39.5GiB | 71.9% done | ETA(epoch): 305s + [Epoch 215][60/66] loss=0.852015 avg=0.842849 VRAM=39.2GiB | 72.0% done | ETA(epoch): 114s + Train loss: 0.838818 (1258.3s) ETA: 1969min + Val loss: 0.846892 [t_0.0-0.2=1.1165 t_0.2-0.4=0.9919 t_0.4-0.6=0.8493 t_0.6-0.8=0.7079 t_0.8-1.0=0.6446] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 216/299 (72% done) --- + [Epoch 216][10/66] loss=0.847430 avg=0.840434 VRAM=39.5GiB | 72.1% done | ETA(epoch): 1066s + [Epoch 216][20/66] loss=0.747937 avg=0.833853 VRAM=39.2GiB | 72.1% done | ETA(epoch): 876s + [Epoch 216][30/66] loss=0.894636 avg=0.844876 VRAM=39.5GiB | 72.2% done | ETA(epoch): 685s + [Epoch 216][40/66] loss=0.852633 avg=0.851570 VRAM=39.2GiB | 72.2% done | ETA(epoch): 495s + [Epoch 216][50/66] loss=0.906569 avg=0.858123 VRAM=39.5GiB | 72.3% done | ETA(epoch): 305s + [Epoch 216][60/66] loss=0.789484 avg=0.859665 VRAM=39.2GiB | 72.3% done | ETA(epoch): 114s + Train loss: 0.858831 (1257.2s) ETA: 1945min + Val loss: 0.864328 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9821 t_0.4-0.6=0.8540 t_0.6-0.8=0.7017 t_0.8-1.0=0.6305] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 217/299 (72% done) --- + [Epoch 217][10/66] loss=0.905489 avg=0.830762 VRAM=39.5GiB | 72.4% done | ETA(epoch): 1065s + [Epoch 217][20/66] loss=0.932715 avg=0.844619 VRAM=39.2GiB | 72.4% done | ETA(epoch): 875s + [Epoch 217][30/66] loss=0.692177 avg=0.851947 VRAM=39.5GiB | 72.5% done | ETA(epoch): 685s + [Epoch 217][40/66] loss=0.926724 avg=0.845405 VRAM=39.2GiB | 72.5% done | ETA(epoch): 495s + [Epoch 217][50/66] loss=0.811764 avg=0.850226 VRAM=39.5GiB | 72.6% done | ETA(epoch): 304s + [Epoch 217][60/66] loss=0.947637 avg=0.852976 VRAM=39.2GiB | 72.6% done | ETA(epoch): 114s + Train loss: 0.852597 (1255.8s) ETA: 1921min + Val loss: 0.856606 [t_0.0-0.2=1.0639 t_0.2-0.4=0.9809 t_0.4-0.6=0.8670 t_0.6-0.8=0.7337 t_0.8-1.0=0.6218] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 218/299 (73% done) --- + [Epoch 218][10/66] loss=0.823652 avg=0.885643 VRAM=39.5GiB | 72.7% done | ETA(epoch): 1066s + [Epoch 218][20/66] loss=0.830605 avg=0.861716 VRAM=39.2GiB | 72.8% done | ETA(epoch): 876s + [Epoch 218][30/66] loss=0.777887 avg=0.842081 VRAM=39.5GiB | 72.8% done | ETA(epoch): 685s + [Epoch 218][40/66] loss=0.830557 avg=0.839144 VRAM=39.2GiB | 72.9% done | ETA(epoch): 495s + [Epoch 218][50/66] loss=0.940253 avg=0.833949 VRAM=39.5GiB | 72.9% done | ETA(epoch): 305s + [Epoch 218][60/66] loss=0.888242 avg=0.841415 VRAM=39.2GiB | 73.0% done | ETA(epoch): 114s + Train loss: 0.840120 (1256.4s) ETA: 1897min + Val loss: 0.830080 [t_0.0-0.2=1.0854 t_0.2-0.4=0.9899 t_0.4-0.6=0.7957 t_0.6-0.8=0.7103 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 219/299 (73% done) --- + [Epoch 219][10/66] loss=0.802884 avg=0.827030 VRAM=39.5GiB | 73.1% done | ETA(epoch): 1065s + [Epoch 219][20/66] loss=0.715721 avg=0.801025 VRAM=39.2GiB | 73.1% done | ETA(epoch): 875s + [Epoch 219][30/66] loss=0.785378 avg=0.824342 VRAM=39.5GiB | 73.2% done | ETA(epoch): 685s + [Epoch 219][40/66] loss=1.027226 avg=0.835640 VRAM=39.2GiB | 73.2% done | ETA(epoch): 496s + [Epoch 219][50/66] loss=0.804768 avg=0.838019 VRAM=39.5GiB | 73.3% done | ETA(epoch): 305s + [Epoch 219][60/66] loss=0.881491 avg=0.847847 VRAM=39.2GiB | 73.3% done | ETA(epoch): 114s + Train loss: 0.850701 (1258.3s) ETA: 1874min + Val loss: 0.847041 [t_0.0-0.2=1.0548 t_0.2-0.4=0.9703 t_0.4-0.6=0.8424 t_0.6-0.8=0.6996 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 21.0/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 220/299 (73% done) --- + [Epoch 220][10/66] loss=0.749417 avg=0.834300 VRAM=39.5GiB | 73.4% done | ETA(epoch): 1066s + [Epoch 220][20/66] loss=0.817837 avg=0.843554 VRAM=39.2GiB | 73.4% done | ETA(epoch): 876s + [Epoch 220][30/66] loss=0.799628 avg=0.849928 VRAM=39.5GiB | 73.5% done | ETA(epoch): 690s + [Epoch 220][40/66] loss=0.822600 avg=0.849108 VRAM=39.2GiB | 73.5% done | ETA(epoch): 498s + [Epoch 220][50/66] loss=0.938757 avg=0.852341 VRAM=39.5GiB | 73.6% done | ETA(epoch): 306s + [Epoch 220][60/66] loss=0.775867 avg=0.851249 VRAM=39.2GiB | 73.6% done | ETA(epoch): 115s + Train loss: 0.847484 (1260.5s) ETA: 1850min + Val loss: 0.836812 [t_0.0-0.2=1.0602 t_0.2-0.4=1.0150 t_0.4-0.6=0.8233 t_0.6-0.8=0.7074 t_0.8-1.0=0.6400] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 221/299 (74% done) --- + [Epoch 221][10/66] loss=0.887961 avg=0.859344 VRAM=39.5GiB | 73.7% done | ETA(epoch): 1066s + [Epoch 221][20/66] loss=0.798777 avg=0.866619 VRAM=39.2GiB | 73.8% done | ETA(epoch): 877s + [Epoch 221][30/66] loss=0.919641 avg=0.857841 VRAM=39.5GiB | 73.8% done | ETA(epoch): 686s + [Epoch 221][40/66] loss=0.840107 avg=0.852353 VRAM=39.2GiB | 73.9% done | ETA(epoch): 496s + [Epoch 221][50/66] loss=0.832682 avg=0.848473 VRAM=39.5GiB | 73.9% done | ETA(epoch): 305s + [Epoch 221][60/66] loss=0.751940 avg=0.848515 VRAM=39.2GiB | 74.0% done | ETA(epoch): 114s + Train loss: 0.844706 (1257.4s) ETA: 1826min + Val loss: 0.847768 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9814 t_0.4-0.6=0.8357 t_0.6-0.8=0.7179 t_0.8-1.0=0.6399] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 222/299 (74% done) --- + [Epoch 222][10/66] loss=0.878960 avg=0.849609 VRAM=39.5GiB | 74.1% done | ETA(epoch): 1065s + [Epoch 222][20/66] loss=0.768522 avg=0.846857 VRAM=39.2GiB | 74.1% done | ETA(epoch): 876s + [Epoch 222][30/66] loss=0.852561 avg=0.853165 VRAM=39.5GiB | 74.2% done | ETA(epoch): 685s + [Epoch 222][40/66] loss=0.967443 avg=0.845201 VRAM=39.2GiB | 74.2% done | ETA(epoch): 495s + [Epoch 222][50/66] loss=0.792401 avg=0.842551 VRAM=39.5GiB | 74.3% done | ETA(epoch): 305s + [Epoch 222][60/66] loss=0.735328 avg=0.838189 VRAM=39.2GiB | 74.3% done | ETA(epoch): 114s + Train loss: 0.842779 (1256.5s) ETA: 1802min + Val loss: 0.877251 [t_0.0-0.2=1.0539 t_0.2-0.4=1.0145 t_0.4-0.6=0.8554 t_0.6-0.8=0.6984 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 223/299 (74% done) --- + [Epoch 223][10/66] loss=0.821472 avg=0.819049 VRAM=39.5GiB | 74.4% done | ETA(epoch): 1081s + [Epoch 223][20/66] loss=0.924087 avg=0.829104 VRAM=39.2GiB | 74.4% done | ETA(epoch): 881s + [Epoch 223][30/66] loss=0.856375 avg=0.845993 VRAM=39.5GiB | 74.5% done | ETA(epoch): 688s + [Epoch 223][40/66] loss=0.927499 avg=0.837317 VRAM=39.2GiB | 74.5% done | ETA(epoch): 496s + [Epoch 223][50/66] loss=0.900745 avg=0.846157 VRAM=39.5GiB | 74.6% done | ETA(epoch): 305s + [Epoch 223][60/66] loss=0.804101 avg=0.839326 VRAM=39.2GiB | 74.6% done | ETA(epoch): 114s + Train loss: 0.843346 (1262.7s) ETA: 1779min + Val loss: 0.842385 [t_0.0-0.2=1.0849 t_0.2-0.4=0.9979 t_0.4-0.6=0.8104 t_0.6-0.8=0.7158 t_0.8-1.0=0.6331] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 224/299 (75% done) --- + [Epoch 224][10/66] loss=0.719650 avg=0.823920 VRAM=39.5GiB | 74.7% done | ETA(epoch): 1065s + [Epoch 224][20/66] loss=0.798486 avg=0.840324 VRAM=39.2GiB | 74.8% done | ETA(epoch): 875s + [Epoch 224][30/66] loss=0.975499 avg=0.847931 VRAM=39.5GiB | 74.8% done | ETA(epoch): 685s + [Epoch 224][40/66] loss=0.866675 avg=0.851757 VRAM=39.2GiB | 74.9% done | ETA(epoch): 495s + [Epoch 224][50/66] loss=0.772162 avg=0.855835 VRAM=39.5GiB | 74.9% done | ETA(epoch): 304s + [Epoch 224][60/66] loss=0.819645 avg=0.849841 VRAM=39.2GiB | 75.0% done | ETA(epoch): 114s + Train loss: 0.852301 (1256.0s) ETA: 1755min + Val loss: 0.838074 [t_0.0-0.2=1.0559 t_0.2-0.4=1.0059 t_0.4-0.6=0.8574 t_0.6-0.8=0.6968 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 225/299 (75% done) --- + [Epoch 225][10/66] loss=0.822321 avg=0.878908 VRAM=39.5GiB | 75.1% done | ETA(epoch): 1066s + [Epoch 225][20/66] loss=0.909580 avg=0.870350 VRAM=39.2GiB | 75.1% done | ETA(epoch): 875s + [Epoch 225][30/66] loss=0.874929 avg=0.867358 VRAM=39.5GiB | 75.2% done | ETA(epoch): 685s + [Epoch 225][40/66] loss=0.890692 avg=0.873572 VRAM=39.2GiB | 75.2% done | ETA(epoch): 495s + [Epoch 225][50/66] loss=0.785497 avg=0.870171 VRAM=39.5GiB | 75.3% done | ETA(epoch): 305s + [Epoch 225][60/66] loss=0.938842 avg=0.874423 VRAM=39.2GiB | 75.3% done | ETA(epoch): 114s + Train loss: 0.872434 (1256.8s) ETA: 1731min + Val loss: 0.868699 [t_0.0-0.2=1.0662 t_0.2-0.4=1.0407 t_0.4-0.6=0.8331 t_0.6-0.8=0.7125 t_0.8-1.0=0.6303] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 21.0/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 226/299 (75% done) --- + [Epoch 226][10/66] loss=0.796935 avg=0.862173 VRAM=39.5GiB | 75.4% done | ETA(epoch): 1064s + [Epoch 226][20/66] loss=0.826095 avg=0.868987 VRAM=39.2GiB | 75.4% done | ETA(epoch): 875s + [Epoch 226][30/66] loss=0.760548 avg=0.863254 VRAM=39.5GiB | 75.5% done | ETA(epoch): 685s + [Epoch 226][40/66] loss=0.883510 avg=0.872507 VRAM=39.2GiB | 75.5% done | ETA(epoch): 495s + [Epoch 226][50/66] loss=0.847736 avg=0.874546 VRAM=39.5GiB | 75.6% done | ETA(epoch): 306s + [Epoch 226][60/66] loss=0.944543 avg=0.874970 VRAM=39.2GiB | 75.6% done | ETA(epoch): 115s + Train loss: 0.870298 (1261.3s) ETA: 1708min + Val loss: 0.842305 [t_0.0-0.2=1.0550 t_0.2-0.4=1.0076 t_0.4-0.6=0.8463 t_0.6-0.8=0.7198 t_0.8-1.0=0.6314] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 227/299 (76% done) --- + [Epoch 227][10/66] loss=0.965280 avg=0.848796 VRAM=39.5GiB | 75.7% done | ETA(epoch): 1065s + [Epoch 227][20/66] loss=0.784075 avg=0.840460 VRAM=39.2GiB | 75.8% done | ETA(epoch): 875s + [Epoch 227][30/66] loss=0.827245 avg=0.843572 VRAM=39.5GiB | 75.8% done | ETA(epoch): 685s + [Epoch 227][40/66] loss=0.828371 avg=0.833972 VRAM=39.2GiB | 75.9% done | ETA(epoch): 496s + [Epoch 227][50/66] loss=0.865920 avg=0.836213 VRAM=39.5GiB | 75.9% done | ETA(epoch): 305s + [Epoch 227][60/66] loss=0.901124 avg=0.845421 VRAM=39.2GiB | 76.0% done | ETA(epoch): 114s + Train loss: 0.846607 (1258.0s) ETA: 1684min + Val loss: 0.826125 [t_0.0-0.2=1.0600 t_0.2-0.4=0.9781 t_0.4-0.6=0.8813 t_0.6-0.8=0.6919 t_0.8-1.0=0.6505] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 228/299 (76% done) --- + [Epoch 228][10/66] loss=0.935606 avg=0.853617 VRAM=39.5GiB | 76.1% done | ETA(epoch): 1065s + [Epoch 228][20/66] loss=0.713433 avg=0.834497 VRAM=39.2GiB | 76.1% done | ETA(epoch): 875s + [Epoch 228][30/66] loss=0.763434 avg=0.841556 VRAM=39.5GiB | 76.2% done | ETA(epoch): 685s + [Epoch 228][40/66] loss=0.761961 avg=0.833495 VRAM=39.2GiB | 76.2% done | ETA(epoch): 495s + [Epoch 228][50/66] loss=0.900090 avg=0.834257 VRAM=39.5GiB | 76.3% done | ETA(epoch): 305s + [Epoch 228][60/66] loss=0.756225 avg=0.833909 VRAM=39.2GiB | 76.3% done | ETA(epoch): 114s + Train loss: 0.832957 (1256.7s) ETA: 1660min + Val loss: 0.859309 [t_0.0-0.2=1.0810 t_0.2-0.4=1.0241 t_0.4-0.6=0.8348 t_0.6-0.8=0.7178 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 229/299 (76% done) --- + [Epoch 229][10/66] loss=0.746033 avg=0.843964 VRAM=39.5GiB | 76.4% done | ETA(epoch): 1067s + [Epoch 229][20/66] loss=0.817201 avg=0.842314 VRAM=39.2GiB | 76.4% done | ETA(epoch): 876s + [Epoch 229][30/66] loss=0.818470 avg=0.847354 VRAM=39.5GiB | 76.5% done | ETA(epoch): 686s + [Epoch 229][40/66] loss=0.794031 avg=0.848845 VRAM=39.2GiB | 76.5% done | ETA(epoch): 495s + [Epoch 229][50/66] loss=0.868862 avg=0.852785 VRAM=39.5GiB | 76.6% done | ETA(epoch): 305s + [Epoch 229][60/66] loss=0.766270 avg=0.853008 VRAM=39.2GiB | 76.6% done | ETA(epoch): 114s + Train loss: 0.851742 (1256.6s) ETA: 1637min + Val loss: 0.838353 [t_0.0-0.2=1.0625 t_0.2-0.4=1.0143 t_0.4-0.6=0.8416 t_0.6-0.8=0.7093 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 230/299 (77% done) --- + [Epoch 230][10/66] loss=0.771170 avg=0.808912 VRAM=39.5GiB | 76.7% done | ETA(epoch): 1065s + [Epoch 230][20/66] loss=0.963794 avg=0.828408 VRAM=39.2GiB | 76.8% done | ETA(epoch): 876s + [Epoch 230][30/66] loss=0.714675 avg=0.833486 VRAM=39.5GiB | 76.8% done | ETA(epoch): 685s + [Epoch 230][40/66] loss=0.899453 avg=0.850773 VRAM=39.2GiB | 76.9% done | ETA(epoch): 495s + [Epoch 230][50/66] loss=0.923165 avg=0.844880 VRAM=39.5GiB | 76.9% done | ETA(epoch): 304s + [Epoch 230][60/66] loss=0.788471 avg=0.845192 VRAM=39.2GiB | 77.0% done | ETA(epoch): 114s + Train loss: 0.849262 (1256.0s) ETA: 1613min + Val loss: 0.831864 [t_0.0-0.2=1.0744 t_0.2-0.4=0.9758 t_0.4-0.6=0.8567 t_0.6-0.8=0.7134 t_0.8-1.0=0.6333] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 231/299 (77% done) --- + [Epoch 231][10/66] loss=0.894032 avg=0.842450 VRAM=39.5GiB | 77.1% done | ETA(epoch): 1078s + [Epoch 231][20/66] loss=0.889707 avg=0.866803 VRAM=39.2GiB | 77.1% done | ETA(epoch): 881s + [Epoch 231][30/66] loss=0.740525 avg=0.857777 VRAM=39.5GiB | 77.2% done | ETA(epoch): 688s + [Epoch 231][40/66] loss=0.764316 avg=0.861300 VRAM=39.2GiB | 77.2% done | ETA(epoch): 496s + [Epoch 231][50/66] loss=0.670805 avg=0.854241 VRAM=39.5GiB | 77.3% done | ETA(epoch): 305s + [Epoch 231][60/66] loss=0.922310 avg=0.854336 VRAM=39.2GiB | 77.3% done | ETA(epoch): 114s + Train loss: 0.856853 (1258.5s) ETA: 1589min + Val loss: 0.849684 [t_0.0-0.2=1.0772 t_0.2-0.4=0.9958 t_0.4-0.6=0.8497 t_0.6-0.8=0.6971 t_0.8-1.0=0.6343] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 21.0/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 232/299 (77% done) --- + [Epoch 232][10/66] loss=0.833860 avg=0.893231 VRAM=39.5GiB | 77.4% done | ETA(epoch): 1065s + [Epoch 232][20/66] loss=0.811680 avg=0.865282 VRAM=39.2GiB | 77.4% done | ETA(epoch): 875s + [Epoch 232][30/66] loss=0.986231 avg=0.852776 VRAM=39.5GiB | 77.5% done | ETA(epoch): 685s + [Epoch 232][40/66] loss=0.982373 avg=0.859346 VRAM=39.2GiB | 77.5% done | ETA(epoch): 495s + [Epoch 232][50/66] loss=0.827236 avg=0.856660 VRAM=39.5GiB | 77.6% done | ETA(epoch): 305s + [Epoch 232][60/66] loss=0.978969 avg=0.855389 VRAM=39.2GiB | 77.6% done | ETA(epoch): 114s + Train loss: 0.851600 (1256.4s) ETA: 1566min + Val loss: 0.842088 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9854 t_0.4-0.6=0.7959 t_0.6-0.8=0.6989 t_0.8-1.0=0.6310] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 233/299 (78% done) --- + [Epoch 233][10/66] loss=0.905854 avg=0.857442 VRAM=39.5GiB | 77.7% done | ETA(epoch): 1067s + [Epoch 233][20/66] loss=0.784290 avg=0.864659 VRAM=39.2GiB | 77.8% done | ETA(epoch): 876s + [Epoch 233][30/66] loss=0.834973 avg=0.874607 VRAM=39.5GiB | 77.8% done | ETA(epoch): 685s + [Epoch 233][40/66] loss=0.821595 avg=0.871198 VRAM=39.2GiB | 77.9% done | ETA(epoch): 495s + [Epoch 233][50/66] loss=0.836366 avg=0.868368 VRAM=39.5GiB | 77.9% done | ETA(epoch): 305s + [Epoch 233][60/66] loss=0.906712 avg=0.867116 VRAM=39.2GiB | 78.0% done | ETA(epoch): 114s + Train loss: 0.867621 (1256.3s) ETA: 1542min + Val loss: 0.860539 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9994 t_0.4-0.6=0.8061 t_0.6-0.8=0.7138 t_0.8-1.0=0.6233] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 234/299 (78% done) --- + [Epoch 234][10/66] loss=0.846613 avg=0.864009 VRAM=39.5GiB | 78.1% done | ETA(epoch): 1065s + [Epoch 234][20/66] loss=0.841770 avg=0.866556 VRAM=39.2GiB | 78.1% done | ETA(epoch): 875s + [Epoch 234][30/66] loss=0.919302 avg=0.861725 VRAM=39.5GiB | 78.2% done | ETA(epoch): 685s + [Epoch 234][40/66] loss=0.944957 avg=0.862528 VRAM=39.2GiB | 78.2% done | ETA(epoch): 495s + [Epoch 234][50/66] loss=0.902764 avg=0.859726 VRAM=39.5GiB | 78.3% done | ETA(epoch): 304s + [Epoch 234][60/66] loss=0.907333 avg=0.858185 VRAM=39.2GiB | 78.3% done | ETA(epoch): 114s + Train loss: 0.859108 (1256.0s) ETA: 1519min + Val loss: 0.837703 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9811 t_0.4-0.6=0.8512 t_0.6-0.8=0.7002 t_0.8-1.0=0.6492] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 21.2/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 235/299 (78% done) --- + [Epoch 235][10/66] loss=0.934740 avg=0.883872 VRAM=39.5GiB | 78.4% done | ETA(epoch): 1065s + [Epoch 235][20/66] loss=0.807526 avg=0.854770 VRAM=39.2GiB | 78.4% done | ETA(epoch): 875s + [Epoch 235][30/66] loss=0.964025 avg=0.860368 VRAM=39.5GiB | 78.5% done | ETA(epoch): 685s + [Epoch 235][40/66] loss=0.909373 avg=0.862495 VRAM=39.2GiB | 78.5% done | ETA(epoch): 496s + [Epoch 235][50/66] loss=0.818643 avg=0.855954 VRAM=39.5GiB | 78.6% done | ETA(epoch): 305s + [Epoch 235][60/66] loss=0.760994 avg=0.849659 VRAM=39.2GiB | 78.6% done | ETA(epoch): 114s + Train loss: 0.852504 (1258.0s) ETA: 1495min + Val loss: 0.854159 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9953 t_0.4-0.6=0.8441 t_0.6-0.8=0.7068 t_0.8-1.0=0.6108] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 236/299 (79% done) --- + [Epoch 236][10/66] loss=0.773240 avg=0.847788 VRAM=39.5GiB | 78.7% done | ETA(epoch): 1066s + [Epoch 236][20/66] loss=0.774560 avg=0.865664 VRAM=39.2GiB | 78.8% done | ETA(epoch): 876s + [Epoch 236][30/66] loss=0.886936 avg=0.866766 VRAM=39.5GiB | 78.8% done | ETA(epoch): 685s + [Epoch 236][40/66] loss=0.840083 avg=0.861952 VRAM=39.2GiB | 78.9% done | ETA(epoch): 495s + [Epoch 236][50/66] loss=1.029825 avg=0.854760 VRAM=39.5GiB | 78.9% done | ETA(epoch): 305s + [Epoch 236][60/66] loss=0.817138 avg=0.850368 VRAM=39.2GiB | 79.0% done | ETA(epoch): 114s + Train loss: 0.851680 (1256.3s) ETA: 1471min + Val loss: 0.882238 [t_0.0-0.2=1.0823 t_0.2-0.4=1.0002 t_0.4-0.6=0.8551 t_0.6-0.8=0.6813 t_0.8-1.0=0.6265] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 237/299 (79% done) --- + [Epoch 237][10/66] loss=0.967732 avg=0.894237 VRAM=39.5GiB | 79.1% done | ETA(epoch): 1066s + [Epoch 237][20/66] loss=0.855021 avg=0.871036 VRAM=39.2GiB | 79.1% done | ETA(epoch): 876s + [Epoch 237][30/66] loss=0.933049 avg=0.881917 VRAM=39.5GiB | 79.2% done | ETA(epoch): 691s + [Epoch 237][40/66] loss=0.829015 avg=0.882731 VRAM=39.2GiB | 79.2% done | ETA(epoch): 498s + [Epoch 237][50/66] loss=0.933120 avg=0.872248 VRAM=39.5GiB | 79.3% done | ETA(epoch): 306s + [Epoch 237][60/66] loss=0.760508 avg=0.864361 VRAM=39.2GiB | 79.3% done | ETA(epoch): 115s + Train loss: 0.862930 (1260.9s) ETA: 1448min + Val loss: 0.860150 [t_0.0-0.2=1.0756 t_0.2-0.4=0.9966 t_0.4-0.6=0.8124 t_0.6-0.8=0.7239 t_0.8-1.0=0.6326] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 238/299 (79% done) --- + [Epoch 238][10/66] loss=0.938234 avg=0.840906 VRAM=39.5GiB | 79.4% done | ETA(epoch): 1065s + [Epoch 238][20/66] loss=0.805373 avg=0.824027 VRAM=39.2GiB | 79.4% done | ETA(epoch): 875s + [Epoch 238][30/66] loss=0.947679 avg=0.835583 VRAM=39.5GiB | 79.5% done | ETA(epoch): 685s + [Epoch 238][40/66] loss=0.734332 avg=0.843784 VRAM=39.2GiB | 79.5% done | ETA(epoch): 495s + [Epoch 238][50/66] loss=0.855986 avg=0.846797 VRAM=39.5GiB | 79.6% done | ETA(epoch): 305s + [Epoch 238][60/66] loss=0.954271 avg=0.848992 VRAM=39.2GiB | 79.6% done | ETA(epoch): 114s + Train loss: 0.841535 (1257.1s) ETA: 1424min + Val loss: 0.839967 [t_0.0-0.2=1.0702 t_0.2-0.4=1.0352 t_0.4-0.6=0.8749 t_0.6-0.8=0.6771 t_0.8-1.0=0.6385] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 239/299 (80% done) --- + [Epoch 239][10/66] loss=0.906421 avg=0.831171 VRAM=39.5GiB | 79.7% done | ETA(epoch): 1067s + [Epoch 239][20/66] loss=0.809885 avg=0.840894 VRAM=39.2GiB | 79.8% done | ETA(epoch): 876s + [Epoch 239][30/66] loss=0.780556 avg=0.852991 VRAM=39.5GiB | 79.8% done | ETA(epoch): 685s + [Epoch 239][40/66] loss=0.788352 avg=0.839966 VRAM=39.2GiB | 79.9% done | ETA(epoch): 495s + [Epoch 239][50/66] loss=0.785752 avg=0.847832 VRAM=39.5GiB | 79.9% done | ETA(epoch): 305s + [Epoch 239][60/66] loss=0.906031 avg=0.848015 VRAM=39.2GiB | 80.0% done | ETA(epoch): 114s + [MilestoneVis] train_0 step 15839 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 15839 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 15839 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 15839 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 15839 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 15839 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 15839 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 15839 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 15839 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 15839 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + Train loss: 0.848733 (2287.1s) ETA: 1405min + Val loss: 0.858990 [t_0.0-0.2=1.0756 t_0.2-0.4=0.9916 t_0.4-0.6=0.8446 t_0.6-0.8=0.7082 t_0.8-1.0=0.6488] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 240/299 (80% done) --- + [Epoch 240][10/66] loss=0.766895 avg=0.796177 VRAM=39.5GiB | 80.1% done | ETA(epoch): 1066s + [Epoch 240][20/66] loss=0.754954 avg=0.822832 VRAM=39.2GiB | 80.1% done | ETA(epoch): 876s + [Epoch 240][30/66] loss=0.752494 avg=0.842230 VRAM=39.5GiB | 80.2% done | ETA(epoch): 686s + [Epoch 240][40/66] loss=0.766058 avg=0.846633 VRAM=39.2GiB | 80.2% done | ETA(epoch): 495s + [Epoch 240][50/66] loss=0.875605 avg=0.856036 VRAM=39.5GiB | 80.3% done | ETA(epoch): 305s + [Epoch 240][60/66] loss=0.732291 avg=0.854206 VRAM=39.2GiB | 80.3% done | ETA(epoch): 114s + Train loss: 0.851762 (1257.4s) ETA: 1381min + Val loss: 0.852164 [t_0.0-0.2=1.0520 t_0.2-0.4=0.9750 t_0.4-0.6=0.8332 t_0.6-0.8=0.7018 t_0.8-1.0=0.6505] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 241/299 (80% done) --- + [Epoch 241][10/66] loss=0.972027 avg=0.883314 VRAM=39.5GiB | 80.4% done | ETA(epoch): 1067s + [Epoch 241][20/66] loss=0.755712 avg=0.871264 VRAM=39.2GiB | 80.4% done | ETA(epoch): 876s + [Epoch 241][30/66] loss=1.023233 avg=0.862321 VRAM=39.5GiB | 80.5% done | ETA(epoch): 685s + [Epoch 241][40/66] loss=0.920068 avg=0.853724 VRAM=39.2GiB | 80.5% done | ETA(epoch): 495s + [Epoch 241][50/66] loss=0.760338 avg=0.850072 VRAM=39.5GiB | 80.6% done | ETA(epoch): 305s + [Epoch 241][60/66] loss=0.816835 avg=0.849313 VRAM=39.2GiB | 80.6% done | ETA(epoch): 114s + Train loss: 0.849484 (1258.6s) ETA: 1358min + Val loss: 0.848630 [t_0.0-0.2=1.0585 t_0.2-0.4=1.0114 t_0.4-0.6=0.8229 t_0.6-0.8=0.7055 t_0.8-1.0=0.6337] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 242/299 (81% done) --- + [Epoch 242][10/66] loss=0.855656 avg=0.829490 VRAM=39.5GiB | 80.7% done | ETA(epoch): 1067s + [Epoch 242][20/66] loss=0.746457 avg=0.825343 VRAM=39.2GiB | 80.8% done | ETA(epoch): 876s + [Epoch 242][30/66] loss=0.920366 avg=0.840563 VRAM=39.5GiB | 80.8% done | ETA(epoch): 685s + [Epoch 242][40/66] loss=0.787733 avg=0.842583 VRAM=39.2GiB | 80.9% done | ETA(epoch): 495s + [Epoch 242][50/66] loss=0.738303 avg=0.839849 VRAM=39.5GiB | 80.9% done | ETA(epoch): 305s + [Epoch 242][60/66] loss=0.723283 avg=0.846842 VRAM=39.2GiB | 81.0% done | ETA(epoch): 114s + Train loss: 0.845956 (1257.0s) ETA: 1334min + Val loss: 0.861783 [t_0.0-0.2=1.0592 t_0.2-0.4=1.0006 t_0.4-0.6=0.8527 t_0.6-0.8=0.7226 t_0.8-1.0=0.6477] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 21.2/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 243/299 (81% done) --- + [Epoch 243][10/66] loss=0.784148 avg=0.835575 VRAM=39.5GiB | 81.1% done | ETA(epoch): 1065s + [Epoch 243][20/66] loss=0.885958 avg=0.845313 VRAM=39.2GiB | 81.1% done | ETA(epoch): 875s + [Epoch 243][30/66] loss=0.995997 avg=0.847877 VRAM=39.5GiB | 81.2% done | ETA(epoch): 685s + [Epoch 243][40/66] loss=0.891389 avg=0.845748 VRAM=39.2GiB | 81.2% done | ETA(epoch): 495s + [Epoch 243][50/66] loss=0.927007 avg=0.849122 VRAM=39.5GiB | 81.3% done | ETA(epoch): 306s + [Epoch 243][60/66] loss=0.904563 avg=0.854685 VRAM=39.2GiB | 81.3% done | ETA(epoch): 115s + Train loss: 0.856311 (1259.5s) ETA: 1311min + Val loss: 0.825948 [t_0.0-0.2=1.0690 t_0.2-0.4=1.0127 t_0.4-0.6=0.8223 t_0.6-0.8=0.7061 t_0.8-1.0=0.6421] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 244/299 (81% done) --- + [Epoch 244][10/66] loss=0.839682 avg=0.838581 VRAM=39.5GiB | 81.4% done | ETA(epoch): 1065s + [Epoch 244][20/66] loss=0.956926 avg=0.840642 VRAM=39.2GiB | 81.4% done | ETA(epoch): 875s + [Epoch 244][30/66] loss=0.730398 avg=0.832261 VRAM=39.5GiB | 81.5% done | ETA(epoch): 685s + [Epoch 244][40/66] loss=0.822713 avg=0.839917 VRAM=39.2GiB | 81.5% done | ETA(epoch): 495s + [Epoch 244][50/66] loss=0.899666 avg=0.840128 VRAM=39.5GiB | 81.6% done | ETA(epoch): 305s + [Epoch 244][60/66] loss=0.882007 avg=0.832708 VRAM=39.2GiB | 81.6% done | ETA(epoch): 114s + Train loss: 0.833261 (1256.8s) ETA: 1287min + Val loss: 0.846586 [t_0.0-0.2=1.0600 t_0.2-0.4=1.0067 t_0.4-0.6=0.8326 t_0.6-0.8=0.7237 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 245/299 (82% done) --- + [Epoch 245][10/66] loss=0.886502 avg=0.863679 VRAM=39.5GiB | 81.7% done | ETA(epoch): 1066s + [Epoch 245][20/66] loss=0.821404 avg=0.843573 VRAM=39.2GiB | 81.8% done | ETA(epoch): 876s + [Epoch 245][30/66] loss=0.833822 avg=0.861999 VRAM=39.5GiB | 81.8% done | ETA(epoch): 685s + [Epoch 245][40/66] loss=0.761130 avg=0.853059 VRAM=39.2GiB | 81.9% done | ETA(epoch): 495s + [Epoch 245][50/66] loss=0.837402 avg=0.843319 VRAM=39.5GiB | 81.9% done | ETA(epoch): 305s + [Epoch 245][60/66] loss=0.808026 avg=0.845285 VRAM=39.2GiB | 82.0% done | ETA(epoch): 114s + Train loss: 0.849144 (1256.4s) ETA: 1263min + Val loss: 0.879502 [t_0.0-0.2=1.0819 t_0.2-0.4=0.9882 t_0.4-0.6=0.8477 t_0.6-0.8=0.7020 t_0.8-1.0=0.6398] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0245 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 246/299 (82% done) --- + [Epoch 246][10/66] loss=0.821310 avg=0.855713 VRAM=39.5GiB | 82.1% done | ETA(epoch): 1066s + [Epoch 246][20/66] loss=0.747565 avg=0.871103 VRAM=39.2GiB | 82.1% done | ETA(epoch): 875s + [Epoch 246][30/66] loss=0.903098 avg=0.851713 VRAM=39.5GiB | 82.2% done | ETA(epoch): 687s + [Epoch 246][40/66] loss=0.778707 avg=0.845581 VRAM=39.2GiB | 82.2% done | ETA(epoch): 496s + [Epoch 246][50/66] loss=0.901628 avg=0.848411 VRAM=39.5GiB | 82.3% done | ETA(epoch): 305s + [Epoch 246][60/66] loss=0.936501 avg=0.849437 VRAM=39.2GiB | 82.3% done | ETA(epoch): 114s + Train loss: 0.846453 (1258.2s) ETA: 1240min + Val loss: 0.819493 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0091 t_0.4-0.6=0.8078 t_0.6-0.8=0.7045 t_0.8-1.0=0.6457] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 247/299 (82% done) --- + [Epoch 247][10/66] loss=0.844284 avg=0.873185 VRAM=39.5GiB | 82.4% done | ETA(epoch): 1066s + [Epoch 247][20/66] loss=0.810741 avg=0.855224 VRAM=39.2GiB | 82.4% done | ETA(epoch): 885s + [Epoch 247][30/66] loss=0.879203 avg=0.855312 VRAM=39.5GiB | 82.5% done | ETA(epoch): 690s + [Epoch 247][40/66] loss=0.812687 avg=0.845904 VRAM=39.2GiB | 82.5% done | ETA(epoch): 498s + [Epoch 247][50/66] loss=0.761570 avg=0.845497 VRAM=39.5GiB | 82.6% done | ETA(epoch): 306s + [Epoch 247][60/66] loss=0.885571 avg=0.845032 VRAM=39.2GiB | 82.6% done | ETA(epoch): 115s + Train loss: 0.847616 (1260.4s) ETA: 1216min + Val loss: 0.844825 [t_0.0-0.2=1.0792 t_0.2-0.4=0.9665 t_0.4-0.6=0.8657 t_0.6-0.8=0.6944 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 248/299 (83% done) --- + [Epoch 248][10/66] loss=0.874790 avg=0.848842 VRAM=39.5GiB | 82.7% done | ETA(epoch): 1077s + [Epoch 248][20/66] loss=0.759735 avg=0.838355 VRAM=39.2GiB | 82.8% done | ETA(epoch): 880s + [Epoch 248][30/66] loss=0.737924 avg=0.834071 VRAM=39.5GiB | 82.8% done | ETA(epoch): 688s + [Epoch 248][40/66] loss=0.814061 avg=0.830173 VRAM=39.2GiB | 82.9% done | ETA(epoch): 496s + [Epoch 248][50/66] loss=0.871881 avg=0.829550 VRAM=39.5GiB | 82.9% done | ETA(epoch): 305s + [Epoch 248][60/66] loss=0.967951 avg=0.833031 VRAM=39.2GiB | 83.0% done | ETA(epoch): 114s + Train loss: 0.832555 (1258.7s) ETA: 1193min + Val loss: 0.836228 [t_0.0-0.2=1.0797 t_0.2-0.4=1.0021 t_0.4-0.6=0.8405 t_0.6-0.8=0.7115 t_0.8-1.0=0.6316] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0248 + Deleted old checkpoint: checkpoint_epoch_0245 +[MEM @ epoch 248 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 249/299 (83% done) --- + [Epoch 249][10/66] loss=0.921885 avg=0.874772 VRAM=39.5GiB | 83.1% done | ETA(epoch): 1066s + [Epoch 249][20/66] loss=0.842099 avg=0.857607 VRAM=39.2GiB | 83.1% done | ETA(epoch): 875s + [Epoch 249][30/66] loss=0.781798 avg=0.858949 VRAM=39.5GiB | 83.2% done | ETA(epoch): 685s + [Epoch 249][40/66] loss=0.729069 avg=0.861746 VRAM=39.2GiB | 83.2% done | ETA(epoch): 495s + [Epoch 249][50/66] loss=0.868404 avg=0.859137 VRAM=39.5GiB | 83.3% done | ETA(epoch): 305s + [Epoch 249][60/66] loss=0.771525 avg=0.856165 VRAM=39.2GiB | 83.3% done | ETA(epoch): 114s + Train loss: 0.859744 (1258.8s) ETA: 1169min + Val loss: 0.877447 [t_0.0-0.2=1.0686 t_0.2-0.4=0.9873 t_0.4-0.6=0.8519 t_0.6-0.8=0.7186 t_0.8-1.0=0.6376] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 250/299 (83% done) --- + [Epoch 250][10/66] loss=0.877744 avg=0.829754 VRAM=39.5GiB | 83.4% done | ETA(epoch): 1067s + [Epoch 250][20/66] loss=0.891367 avg=0.821037 VRAM=39.2GiB | 83.4% done | ETA(epoch): 877s + [Epoch 250][30/66] loss=0.827866 avg=0.835945 VRAM=39.5GiB | 83.5% done | ETA(epoch): 686s + [Epoch 250][40/66] loss=0.956638 avg=0.849358 VRAM=39.2GiB | 83.5% done | ETA(epoch): 496s + [Epoch 250][50/66] loss=0.931343 avg=0.849142 VRAM=39.5GiB | 83.6% done | ETA(epoch): 305s + [Epoch 250][60/66] loss=0.888513 avg=0.849938 VRAM=39.2GiB | 83.6% done | ETA(epoch): 114s + Train loss: 0.850590 (1258.0s) ETA: 1146min + Val loss: 0.846719 [t_0.0-0.2=1.0634 t_0.2-0.4=1.0236 t_0.4-0.6=0.8598 t_0.6-0.8=0.7136 t_0.8-1.0=0.6372] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 251/299 (84% done) --- + [Epoch 251][10/66] loss=0.996224 avg=0.833921 VRAM=39.5GiB | 83.7% done | ETA(epoch): 1065s + [Epoch 251][20/66] loss=0.775298 avg=0.853402 VRAM=39.2GiB | 83.8% done | ETA(epoch): 876s + [Epoch 251][30/66] loss=0.853459 avg=0.844499 VRAM=39.5GiB | 83.8% done | ETA(epoch): 685s + [Epoch 251][40/66] loss=0.876679 avg=0.857576 VRAM=39.2GiB | 83.9% done | ETA(epoch): 495s + [Epoch 251][50/66] loss=0.842013 avg=0.855782 VRAM=39.5GiB | 83.9% done | ETA(epoch): 305s + [Epoch 251][60/66] loss=0.841502 avg=0.851219 VRAM=39.2GiB | 84.0% done | ETA(epoch): 114s + Train loss: 0.854306 (1258.9s) ETA: 1122min + Val loss: 0.870631 [t_0.0-0.2=1.0782 t_0.2-0.4=0.9827 t_0.4-0.6=0.8173 t_0.6-0.8=0.7294 t_0.8-1.0=0.6329] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 252/299 (84% done) --- + [Epoch 252][10/66] loss=0.740688 avg=0.846846 VRAM=39.5GiB | 84.1% done | ETA(epoch): 1067s + [Epoch 252][20/66] loss=0.808716 avg=0.855152 VRAM=39.2GiB | 84.1% done | ETA(epoch): 876s + [Epoch 252][30/66] loss=0.793675 avg=0.859786 VRAM=39.5GiB | 84.2% done | ETA(epoch): 686s + [Epoch 252][40/66] loss=0.886985 avg=0.860292 VRAM=39.2GiB | 84.2% done | ETA(epoch): 495s + [Epoch 252][50/66] loss=0.843703 avg=0.856440 VRAM=39.5GiB | 84.3% done | ETA(epoch): 305s + [Epoch 252][60/66] loss=0.829722 avg=0.856741 VRAM=39.2GiB | 84.3% done | ETA(epoch): 114s + Train loss: 0.853049 (1256.5s) ETA: 1098min + Val loss: 0.847755 [t_0.0-0.2=1.0721 t_0.2-0.4=1.0152 t_0.4-0.6=0.8652 t_0.6-0.8=0.6832 t_0.8-1.0=0.6398] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 253/299 (84% done) --- + [Epoch 253][10/66] loss=0.735446 avg=0.846319 VRAM=39.5GiB | 84.4% done | ETA(epoch): 1065s + [Epoch 253][20/66] loss=0.916887 avg=0.858843 VRAM=39.2GiB | 84.4% done | ETA(epoch): 876s + [Epoch 253][30/66] loss=0.896061 avg=0.860011 VRAM=39.5GiB | 84.5% done | ETA(epoch): 685s + [Epoch 253][40/66] loss=0.787534 avg=0.861832 VRAM=39.2GiB | 84.5% done | ETA(epoch): 495s + [Epoch 253][50/66] loss=0.732439 avg=0.863125 VRAM=39.5GiB | 84.6% done | ETA(epoch): 305s + [Epoch 253][60/66] loss=0.755700 avg=0.856856 VRAM=39.2GiB | 84.6% done | ETA(epoch): 114s + Train loss: 0.857044 (1256.8s) ETA: 1075min + Val loss: 0.847413 [t_0.0-0.2=1.0648 t_0.2-0.4=0.9697 t_0.4-0.6=0.8363 t_0.6-0.8=0.7128 t_0.8-1.0=0.6545] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 254/299 (85% done) --- + [Epoch 254][10/66] loss=0.866402 avg=0.825372 VRAM=39.5GiB | 84.7% done | ETA(epoch): 1067s + [Epoch 254][20/66] loss=0.937801 avg=0.827639 VRAM=39.2GiB | 84.8% done | ETA(epoch): 876s + [Epoch 254][30/66] loss=0.734610 avg=0.831417 VRAM=39.5GiB | 84.8% done | ETA(epoch): 690s + [Epoch 254][40/66] loss=0.757444 avg=0.831509 VRAM=39.2GiB | 84.9% done | ETA(epoch): 498s + [Epoch 254][50/66] loss=0.922157 avg=0.840431 VRAM=39.5GiB | 84.9% done | ETA(epoch): 306s + [Epoch 254][60/66] loss=0.986440 avg=0.836831 VRAM=39.2GiB | 85.0% done | ETA(epoch): 115s + Train loss: 0.838561 (1260.3s) ETA: 1051min + Val loss: 0.880593 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9980 t_0.4-0.6=0.8007 t_0.6-0.8=0.7249 t_0.8-1.0=0.6098] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 255/299 (85% done) --- + [Epoch 255][10/66] loss=0.733953 avg=0.828230 VRAM=39.5GiB | 85.1% done | ETA(epoch): 1065s + [Epoch 255][20/66] loss=0.893929 avg=0.842794 VRAM=39.2GiB | 85.1% done | ETA(epoch): 880s + [Epoch 255][30/66] loss=0.801667 avg=0.840668 VRAM=39.5GiB | 85.2% done | ETA(epoch): 688s + [Epoch 255][40/66] loss=0.955948 avg=0.840739 VRAM=39.2GiB | 85.2% done | ETA(epoch): 496s + [Epoch 255][50/66] loss=0.918788 avg=0.840569 VRAM=39.5GiB | 85.3% done | ETA(epoch): 305s + [Epoch 255][60/66] loss=0.800894 avg=0.844265 VRAM=39.2GiB | 85.3% done | ETA(epoch): 114s + Train loss: 0.852860 (1258.7s) ETA: 1028min + Val loss: 0.841189 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9959 t_0.4-0.6=0.8241 t_0.6-0.8=0.7041 t_0.8-1.0=0.6359] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 256/299 (85% done) --- + [Epoch 256][10/66] loss=0.852449 avg=0.880196 VRAM=39.5GiB | 85.4% done | ETA(epoch): 1064s + [Epoch 256][20/66] loss=0.739324 avg=0.859753 VRAM=39.2GiB | 85.4% done | ETA(epoch): 875s + [Epoch 256][30/66] loss=0.842283 avg=0.849486 VRAM=39.5GiB | 85.5% done | ETA(epoch): 685s + [Epoch 256][40/66] loss=0.794958 avg=0.860336 VRAM=39.2GiB | 85.5% done | ETA(epoch): 495s + [Epoch 256][50/66] loss=0.847377 avg=0.860599 VRAM=39.5GiB | 85.6% done | ETA(epoch): 305s + [Epoch 256][60/66] loss=0.822066 avg=0.859766 VRAM=39.2GiB | 85.6% done | ETA(epoch): 114s + Train loss: 0.861242 (1256.0s) ETA: 1004min + Val loss: 0.838202 [t_0.0-0.2=1.0666 t_0.2-0.4=1.0007 t_0.4-0.6=0.8605 t_0.6-0.8=0.6767 t_0.8-1.0=0.6231] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 257/299 (86% done) --- + [Epoch 257][10/66] loss=0.903048 avg=0.883842 VRAM=39.5GiB | 85.7% done | ETA(epoch): 1102s + [Epoch 257][20/66] loss=0.807797 avg=0.874900 VRAM=39.2GiB | 85.8% done | ETA(epoch): 890s + [Epoch 257][30/66] loss=0.884914 avg=0.870077 VRAM=39.5GiB | 85.8% done | ETA(epoch): 693s + [Epoch 257][40/66] loss=0.893125 avg=0.871401 VRAM=39.2GiB | 85.9% done | ETA(epoch): 499s + [Epoch 257][50/66] loss=0.861888 avg=0.870561 VRAM=39.5GiB | 85.9% done | ETA(epoch): 307s + [Epoch 257][60/66] loss=0.918423 avg=0.869270 VRAM=39.2GiB | 86.0% done | ETA(epoch): 115s + Train loss: 0.869292 (1264.7s) ETA: 981min + Val loss: 0.839188 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9469 t_0.4-0.6=0.8723 t_0.6-0.8=0.6970 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 258/299 (86% done) --- + [Epoch 258][10/66] loss=0.769321 avg=0.856426 VRAM=39.5GiB | 86.1% done | ETA(epoch): 1065s + [Epoch 258][20/66] loss=0.919506 avg=0.873737 VRAM=39.2GiB | 86.1% done | ETA(epoch): 876s + [Epoch 258][30/66] loss=0.879698 avg=0.863344 VRAM=39.5GiB | 86.2% done | ETA(epoch): 686s + [Epoch 258][40/66] loss=0.815089 avg=0.861808 VRAM=39.2GiB | 86.2% done | ETA(epoch): 495s + [Epoch 258][50/66] loss=0.862209 avg=0.856634 VRAM=39.5GiB | 86.3% done | ETA(epoch): 305s + [Epoch 258][60/66] loss=0.881588 avg=0.846077 VRAM=39.2GiB | 86.3% done | ETA(epoch): 114s + Train loss: 0.849684 (1258.4s) ETA: 957min + Val loss: 0.855192 [t_0.0-0.2=1.0873 t_0.2-0.4=1.0047 t_0.4-0.6=0.8295 t_0.6-0.8=0.7147 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 259/299 (86% done) --- + [Epoch 259][10/66] loss=0.856977 avg=0.865043 VRAM=39.5GiB | 86.4% done | ETA(epoch): 1063s + [Epoch 259][20/66] loss=0.712829 avg=0.840220 VRAM=39.2GiB | 86.4% done | ETA(epoch): 874s + [Epoch 259][30/66] loss=0.871048 avg=0.837118 VRAM=39.5GiB | 86.5% done | ETA(epoch): 684s + [Epoch 259][40/66] loss=0.823939 avg=0.836895 VRAM=39.2GiB | 86.5% done | ETA(epoch): 494s + [Epoch 259][50/66] loss=0.845426 avg=0.843424 VRAM=39.5GiB | 86.6% done | ETA(epoch): 304s + [Epoch 259][60/66] loss=0.899160 avg=0.845370 VRAM=39.2GiB | 86.6% done | ETA(epoch): 114s + Train loss: 0.852675 (1254.9s) ETA: 934min + Val loss: 0.826577 [t_0.0-0.2=1.0604 t_0.2-0.4=0.9883 t_0.4-0.6=0.8584 t_0.6-0.8=0.6994 t_0.8-1.0=0.6303] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 260/299 (87% done) --- + [Epoch 260][10/66] loss=0.832890 avg=0.827590 VRAM=39.5GiB | 86.7% done | ETA(epoch): 1064s + [Epoch 260][20/66] loss=0.905383 avg=0.836211 VRAM=39.2GiB | 86.8% done | ETA(epoch): 875s + [Epoch 260][30/66] loss=0.679451 avg=0.843590 VRAM=39.5GiB | 86.8% done | ETA(epoch): 685s + [Epoch 260][40/66] loss=0.961960 avg=0.855870 VRAM=39.2GiB | 86.9% done | ETA(epoch): 495s + [Epoch 260][50/66] loss=0.933457 avg=0.854345 VRAM=39.5GiB | 86.9% done | ETA(epoch): 306s + [Epoch 260][60/66] loss=0.834258 avg=0.854954 VRAM=39.2GiB | 87.0% done | ETA(epoch): 115s + Train loss: 0.852757 (1260.7s) ETA: 911min + Val loss: 0.853494 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9981 t_0.4-0.6=0.8524 t_0.6-0.8=0.7297 t_0.8-1.0=0.6258] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 261/299 (87% done) --- + [Epoch 261][10/66] loss=0.849091 avg=0.857507 VRAM=39.5GiB | 87.1% done | ETA(epoch): 1066s + [Epoch 261][20/66] loss=0.885728 avg=0.853087 VRAM=39.2GiB | 87.1% done | ETA(epoch): 876s + [Epoch 261][30/66] loss=1.026093 avg=0.852972 VRAM=39.5GiB | 87.2% done | ETA(epoch): 685s + [Epoch 261][40/66] loss=0.790359 avg=0.863416 VRAM=39.2GiB | 87.2% done | ETA(epoch): 495s + [Epoch 261][50/66] loss=0.746225 avg=0.857010 VRAM=39.5GiB | 87.3% done | ETA(epoch): 305s + [Epoch 261][60/66] loss=0.873604 avg=0.856325 VRAM=39.2GiB | 87.3% done | ETA(epoch): 114s + Train loss: 0.859376 (1256.9s) ETA: 887min + Val loss: 0.847375 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9903 t_0.4-0.6=0.8437 t_0.6-0.8=0.7080 t_0.8-1.0=0.6350] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 262/299 (87% done) --- + [Epoch 262][10/66] loss=0.730375 avg=0.808270 VRAM=39.5GiB | 87.4% done | ETA(epoch): 1067s + [Epoch 262][20/66] loss=0.859896 avg=0.839289 VRAM=39.2GiB | 87.4% done | ETA(epoch): 876s + [Epoch 262][30/66] loss=0.856619 avg=0.833175 VRAM=39.5GiB | 87.5% done | ETA(epoch): 688s + [Epoch 262][40/66] loss=0.955348 avg=0.836269 VRAM=39.2GiB | 87.5% done | ETA(epoch): 497s + [Epoch 262][50/66] loss=0.795484 avg=0.838557 VRAM=39.5GiB | 87.6% done | ETA(epoch): 305s + [Epoch 262][60/66] loss=0.861089 avg=0.846027 VRAM=39.2GiB | 87.6% done | ETA(epoch): 114s + Train loss: 0.842179 (1259.1s) ETA: 864min + Val loss: 0.841361 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9896 t_0.4-0.6=0.8359 t_0.6-0.8=0.7182 t_0.8-1.0=0.6213] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 263/299 (88% done) --- + [Epoch 263][10/66] loss=0.690979 avg=0.816945 VRAM=39.5GiB | 87.7% done | ETA(epoch): 1067s + [Epoch 263][20/66] loss=0.806638 avg=0.835386 VRAM=39.2GiB | 87.8% done | ETA(epoch): 877s + [Epoch 263][30/66] loss=0.900759 avg=0.844771 VRAM=39.5GiB | 87.8% done | ETA(epoch): 686s + [Epoch 263][40/66] loss=0.754927 avg=0.838663 VRAM=39.2GiB | 87.9% done | ETA(epoch): 495s + [Epoch 263][50/66] loss=0.745233 avg=0.836618 VRAM=39.5GiB | 87.9% done | ETA(epoch): 305s + [Epoch 263][60/66] loss=0.906254 avg=0.839326 VRAM=39.2GiB | 88.0% done | ETA(epoch): 114s + Train loss: 0.841556 (1257.6s) ETA: 840min + Val loss: 0.839323 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9835 t_0.4-0.6=0.8246 t_0.6-0.8=0.7071 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 264/299 (88% done) --- + [Epoch 264][10/66] loss=0.823252 avg=0.834880 VRAM=39.5GiB | 88.1% done | ETA(epoch): 1067s + [Epoch 264][20/66] loss=0.674107 avg=0.838749 VRAM=39.2GiB | 88.1% done | ETA(epoch): 876s + [Epoch 264][30/66] loss=0.923472 avg=0.847167 VRAM=39.5GiB | 88.2% done | ETA(epoch): 686s + [Epoch 264][40/66] loss=0.887099 avg=0.857478 VRAM=39.2GiB | 88.2% done | ETA(epoch): 495s + [Epoch 264][50/66] loss=0.632424 avg=0.850271 VRAM=39.5GiB | 88.3% done | ETA(epoch): 305s + [Epoch 264][60/66] loss=0.873545 avg=0.849456 VRAM=39.2GiB | 88.3% done | ETA(epoch): 114s + Train loss: 0.850760 (1257.8s) ETA: 817min + Val loss: 0.841206 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9638 t_0.4-0.6=0.8348 t_0.6-0.8=0.7242 t_0.8-1.0=0.6284] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 265/299 (88% done) --- + [Epoch 265][10/66] loss=0.904014 avg=0.851747 VRAM=39.5GiB | 88.4% done | ETA(epoch): 1076s + [Epoch 265][20/66] loss=0.798925 avg=0.846157 VRAM=39.2GiB | 88.4% done | ETA(epoch): 880s + [Epoch 265][30/66] loss=0.938361 avg=0.849893 VRAM=39.5GiB | 88.5% done | ETA(epoch): 688s + [Epoch 265][40/66] loss=0.889615 avg=0.855766 VRAM=39.2GiB | 88.5% done | ETA(epoch): 496s + [Epoch 265][50/66] loss=0.916770 avg=0.850479 VRAM=39.5GiB | 88.6% done | ETA(epoch): 305s + [Epoch 265][60/66] loss=0.795359 avg=0.852282 VRAM=39.2GiB | 88.6% done | ETA(epoch): 114s + Train loss: 0.851598 (1259.0s) ETA: 793min + Val loss: 0.850028 [t_0.0-0.2=1.0766 t_0.2-0.4=1.0029 t_0.4-0.6=0.8007 t_0.6-0.8=0.7037 t_0.8-1.0=0.6246] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 266/299 (89% done) --- + [Epoch 266][10/66] loss=0.827052 avg=0.859775 VRAM=39.5GiB | 88.7% done | ETA(epoch): 1066s + [Epoch 266][20/66] loss=0.767848 avg=0.851441 VRAM=39.2GiB | 88.8% done | ETA(epoch): 876s + [Epoch 266][30/66] loss=0.853621 avg=0.852504 VRAM=39.5GiB | 88.8% done | ETA(epoch): 685s + [Epoch 266][40/66] loss=0.952795 avg=0.857362 VRAM=39.2GiB | 88.9% done | ETA(epoch): 495s + [Epoch 266][50/66] loss=0.766279 avg=0.850580 VRAM=39.5GiB | 88.9% done | ETA(epoch): 305s + [Epoch 266][60/66] loss=0.773927 avg=0.846582 VRAM=39.2GiB | 89.0% done | ETA(epoch): 114s + Train loss: 0.850771 (1256.9s) ETA: 770min + Val loss: 0.862545 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9663 t_0.4-0.6=0.8419 t_0.6-0.8=0.7113 t_0.8-1.0=0.6274] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 267/299 (89% done) --- + [Epoch 267][10/66] loss=0.789332 avg=0.804226 VRAM=39.5GiB | 89.1% done | ETA(epoch): 1068s + [Epoch 267][20/66] loss=0.902507 avg=0.834856 VRAM=39.2GiB | 89.1% done | ETA(epoch): 876s + [Epoch 267][30/66] loss=0.881664 avg=0.847655 VRAM=39.5GiB | 89.2% done | ETA(epoch): 685s + [Epoch 267][40/66] loss=0.912825 avg=0.860106 VRAM=39.2GiB | 89.2% done | ETA(epoch): 495s + [Epoch 267][50/66] loss=0.846482 avg=0.856211 VRAM=39.5GiB | 89.3% done | ETA(epoch): 305s + [Epoch 267][60/66] loss=0.832422 avg=0.851658 VRAM=39.2GiB | 89.3% done | ETA(epoch): 115s + Train loss: 0.846976 (1260.2s) ETA: 746min + Val loss: 0.871545 [t_0.0-0.2=1.0733 t_0.2-0.4=0.9890 t_0.4-0.6=0.8533 t_0.6-0.8=0.7227 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 268/299 (89% done) --- + [Epoch 268][10/66] loss=0.800770 avg=0.852847 VRAM=39.5GiB | 89.4% done | ETA(epoch): 1068s + [Epoch 268][20/66] loss=0.855149 avg=0.846828 VRAM=39.2GiB | 89.4% done | ETA(epoch): 877s + [Epoch 268][30/66] loss=0.908442 avg=0.854358 VRAM=39.5GiB | 89.5% done | ETA(epoch): 686s + [Epoch 268][40/66] loss=0.772861 avg=0.854846 VRAM=39.2GiB | 89.5% done | ETA(epoch): 496s + [Epoch 268][50/66] loss=0.874019 avg=0.858426 VRAM=39.5GiB | 89.6% done | ETA(epoch): 306s + [Epoch 268][60/66] loss=0.828881 avg=0.854596 VRAM=39.2GiB | 89.6% done | ETA(epoch): 115s + Train loss: 0.855927 (1260.6s) ETA: 723min + Val loss: 0.864015 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9650 t_0.4-0.6=0.8650 t_0.6-0.8=0.6789 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 269/299 (90% done) --- + [Epoch 269][10/66] loss=0.836177 avg=0.849464 VRAM=39.5GiB | 89.7% done | ETA(epoch): 1066s + [Epoch 269][20/66] loss=0.951586 avg=0.855281 VRAM=39.2GiB | 89.8% done | ETA(epoch): 876s + [Epoch 269][30/66] loss=1.021903 avg=0.859701 VRAM=39.5GiB | 89.8% done | ETA(epoch): 686s + [Epoch 269][40/66] loss=0.860412 avg=0.868168 VRAM=39.2GiB | 89.9% done | ETA(epoch): 496s + [Epoch 269][50/66] loss=0.851149 avg=0.869092 VRAM=39.5GiB | 89.9% done | ETA(epoch): 305s + [Epoch 269][60/66] loss=0.893134 avg=0.865698 VRAM=39.2GiB | 90.0% done | ETA(epoch): 114s + [MilestoneVis] train_0 step 17819 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 17819 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 17819 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 17819 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 17819 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 17819 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 17819 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 17819 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 17819 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 17819 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + Train loss: 0.867359 (2290.3s) ETA: 702min + Val loss: 0.860562 [t_0.0-0.2=1.0639 t_0.2-0.4=1.0141 t_0.4-0.6=0.8289 t_0.6-0.8=0.7150 t_0.8-1.0=0.6444] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 270/299 (90% done) --- + [Epoch 270][10/66] loss=0.907800 avg=0.887936 VRAM=39.5GiB | 90.1% done | ETA(epoch): 1065s + [Epoch 270][20/66] loss=0.855028 avg=0.875914 VRAM=39.2GiB | 90.1% done | ETA(epoch): 875s + [Epoch 270][30/66] loss=0.887807 avg=0.869143 VRAM=39.5GiB | 90.2% done | ETA(epoch): 685s + [Epoch 270][40/66] loss=0.875809 avg=0.865762 VRAM=39.2GiB | 90.2% done | ETA(epoch): 495s + [Epoch 270][50/66] loss=0.783786 avg=0.859163 VRAM=39.5GiB | 90.3% done | ETA(epoch): 305s + [Epoch 270][60/66] loss=0.881514 avg=0.855673 VRAM=39.2GiB | 90.3% done | ETA(epoch): 114s + Train loss: 0.852271 (1256.5s) ETA: 678min + Val loss: 0.858701 [t_0.0-0.2=1.0854 t_0.2-0.4=1.0018 t_0.4-0.6=0.8565 t_0.6-0.8=0.7068 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 271/299 (90% done) --- + [Epoch 271][10/66] loss=0.939894 avg=0.840871 VRAM=39.5GiB | 90.4% done | ETA(epoch): 1066s + [Epoch 271][20/66] loss=0.756031 avg=0.847509 VRAM=39.2GiB | 90.4% done | ETA(epoch): 876s + [Epoch 271][30/66] loss=0.808410 avg=0.849830 VRAM=39.5GiB | 90.5% done | ETA(epoch): 690s + [Epoch 271][40/66] loss=0.846560 avg=0.848881 VRAM=39.2GiB | 90.5% done | ETA(epoch): 498s + [Epoch 271][50/66] loss=0.814996 avg=0.851347 VRAM=39.5GiB | 90.6% done | ETA(epoch): 306s + [Epoch 271][60/66] loss=0.894576 avg=0.859673 VRAM=39.2GiB | 90.6% done | ETA(epoch): 115s + Train loss: 0.861313 (1260.7s) ETA: 655min + Val loss: 0.837655 [t_0.0-0.2=1.0917 t_0.2-0.4=0.9924 t_0.4-0.6=0.8192 t_0.6-0.8=0.7003 t_0.8-1.0=0.6165] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 272/299 (91% done) --- + [Epoch 272][10/66] loss=0.797765 avg=0.846931 VRAM=39.5GiB | 90.7% done | ETA(epoch): 1066s + [Epoch 272][20/66] loss=0.897330 avg=0.839686 VRAM=39.2GiB | 90.8% done | ETA(epoch): 876s + [Epoch 272][30/66] loss=0.887477 avg=0.849467 VRAM=39.5GiB | 90.8% done | ETA(epoch): 688s + [Epoch 272][40/66] loss=0.952982 avg=0.853708 VRAM=39.2GiB | 90.9% done | ETA(epoch): 496s + [Epoch 272][50/66] loss=0.779746 avg=0.854603 VRAM=39.5GiB | 90.9% done | ETA(epoch): 305s + [Epoch 272][60/66] loss=0.944134 avg=0.853222 VRAM=39.2GiB | 91.0% done | ETA(epoch): 114s + Train loss: 0.854516 (1258.7s) ETA: 631min + Val loss: 0.857948 [t_0.0-0.2=1.0957 t_0.2-0.4=0.9207 t_0.4-0.6=0.8387 t_0.6-0.8=0.7344 t_0.8-1.0=0.6336] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 273/299 (91% done) --- + [Epoch 273][10/66] loss=0.846950 avg=0.844342 VRAM=39.5GiB | 91.1% done | ETA(epoch): 1066s + [Epoch 273][20/66] loss=0.747193 avg=0.832309 VRAM=39.2GiB | 91.1% done | ETA(epoch): 880s + [Epoch 273][30/66] loss=0.714038 avg=0.827802 VRAM=39.5GiB | 91.2% done | ETA(epoch): 688s + [Epoch 273][40/66] loss=0.835428 avg=0.837104 VRAM=39.2GiB | 91.2% done | ETA(epoch): 496s + [Epoch 273][50/66] loss=0.860484 avg=0.840496 VRAM=39.5GiB | 91.3% done | ETA(epoch): 305s + [Epoch 273][60/66] loss=0.983368 avg=0.848759 VRAM=39.2GiB | 91.3% done | ETA(epoch): 114s + Train loss: 0.851979 (1259.1s) ETA: 608min + Val loss: 0.820830 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9760 t_0.4-0.6=0.8313 t_0.6-0.8=0.6910 t_0.8-1.0=0.6285] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 274/299 (91% done) --- + [Epoch 274][10/66] loss=0.754873 avg=0.862396 VRAM=39.5GiB | 91.4% done | ETA(epoch): 1066s + [Epoch 274][20/66] loss=0.882707 avg=0.857328 VRAM=39.2GiB | 91.4% done | ETA(epoch): 876s + [Epoch 274][30/66] loss=0.835321 avg=0.845421 VRAM=39.5GiB | 91.5% done | ETA(epoch): 686s + [Epoch 274][40/66] loss=0.725133 avg=0.847537 VRAM=39.2GiB | 91.5% done | ETA(epoch): 495s + [Epoch 274][50/66] loss=0.822526 avg=0.844185 VRAM=39.5GiB | 91.6% done | ETA(epoch): 305s + [Epoch 274][60/66] loss=0.931617 avg=0.854565 VRAM=39.2GiB | 91.6% done | ETA(epoch): 114s + Train loss: 0.850866 (1257.8s) ETA: 584min + Val loss: 0.846135 [t_0.0-0.2=1.0683 t_0.2-0.4=0.9635 t_0.4-0.6=0.8158 t_0.6-0.8=0.7241 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 275/299 (92% done) --- + [Epoch 275][10/66] loss=0.955656 avg=0.870515 VRAM=39.5GiB | 91.7% done | ETA(epoch): 1066s + [Epoch 275][20/66] loss=0.901804 avg=0.842145 VRAM=39.2GiB | 91.8% done | ETA(epoch): 876s + [Epoch 275][30/66] loss=0.755482 avg=0.832316 VRAM=39.5GiB | 91.8% done | ETA(epoch): 686s + [Epoch 275][40/66] loss=0.850978 avg=0.842595 VRAM=39.2GiB | 91.9% done | ETA(epoch): 495s + [Epoch 275][50/66] loss=0.873654 avg=0.847772 VRAM=39.5GiB | 91.9% done | ETA(epoch): 305s + [Epoch 275][60/66] loss=0.907037 avg=0.856745 VRAM=39.2GiB | 92.0% done | ETA(epoch): 114s + Train loss: 0.860016 (1258.9s) ETA: 561min + Val loss: 0.855417 [t_0.0-0.2=1.0625 t_0.2-0.4=0.9868 t_0.4-0.6=0.8474 t_0.6-0.8=0.7105 t_0.8-1.0=0.6305] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 276/299 (92% done) --- + [Epoch 276][10/66] loss=0.898614 avg=0.890585 VRAM=39.5GiB | 92.1% done | ETA(epoch): 1065s + [Epoch 276][20/66] loss=0.863958 avg=0.866385 VRAM=39.2GiB | 92.1% done | ETA(epoch): 875s + [Epoch 276][30/66] loss=0.914950 avg=0.855815 VRAM=39.5GiB | 92.2% done | ETA(epoch): 685s + [Epoch 276][40/66] loss=0.870345 avg=0.852592 VRAM=39.2GiB | 92.2% done | ETA(epoch): 495s + [Epoch 276][50/66] loss=1.014847 avg=0.850612 VRAM=39.5GiB | 92.3% done | ETA(epoch): 305s + [Epoch 276][60/66] loss=0.868179 avg=0.858538 VRAM=39.2GiB | 92.3% done | ETA(epoch): 114s + Train loss: 0.861499 (1256.4s) ETA: 537min + Val loss: 0.856588 [t_0.0-0.2=1.0658 t_0.2-0.4=1.0043 t_0.4-0.6=0.8663 t_0.6-0.8=0.6957 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 277/299 (92% done) --- + [Epoch 277][10/66] loss=0.838389 avg=0.856409 VRAM=39.5GiB | 92.4% done | ETA(epoch): 1065s + [Epoch 277][20/66] loss=0.771766 avg=0.849400 VRAM=39.2GiB | 92.4% done | ETA(epoch): 876s + [Epoch 277][30/66] loss=0.824095 avg=0.859319 VRAM=39.5GiB | 92.5% done | ETA(epoch): 686s + [Epoch 277][40/66] loss=0.891741 avg=0.857546 VRAM=39.2GiB | 92.5% done | ETA(epoch): 495s + [Epoch 277][50/66] loss=0.739986 avg=0.847920 VRAM=39.5GiB | 92.6% done | ETA(epoch): 306s + [Epoch 277][60/66] loss=0.883283 avg=0.848324 VRAM=39.2GiB | 92.6% done | ETA(epoch): 115s + Train loss: 0.849801 (1261.5s) ETA: 514min + Val loss: 0.836204 [t_0.0-0.2=1.0613 t_0.2-0.4=1.0023 t_0.4-0.6=0.8358 t_0.6-0.8=0.7124 t_0.8-1.0=0.6337] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 278/299 (93% done) --- + [Epoch 278][10/66] loss=0.821067 avg=0.866868 VRAM=39.5GiB | 92.7% done | ETA(epoch): 1066s + [Epoch 278][20/66] loss=0.810988 avg=0.871525 VRAM=39.2GiB | 92.8% done | ETA(epoch): 876s + [Epoch 278][30/66] loss=0.994386 avg=0.862079 VRAM=39.5GiB | 92.8% done | ETA(epoch): 686s + [Epoch 278][40/66] loss=0.974768 avg=0.860551 VRAM=39.2GiB | 92.9% done | ETA(epoch): 497s + [Epoch 278][50/66] loss=0.823915 avg=0.865789 VRAM=39.5GiB | 92.9% done | ETA(epoch): 306s + [Epoch 278][60/66] loss=0.888233 avg=0.863680 VRAM=39.2GiB | 93.0% done | ETA(epoch): 115s + Train loss: 0.864693 (1260.0s) ETA: 491min + Val loss: 0.861334 [t_0.0-0.2=1.0389 t_0.2-0.4=0.9867 t_0.4-0.6=0.8551 t_0.6-0.8=0.7129 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 279/299 (93% done) --- + [Epoch 279][10/66] loss=0.861076 avg=0.848386 VRAM=39.5GiB | 93.1% done | ETA(epoch): 1066s + [Epoch 279][20/66] loss=0.828229 avg=0.853056 VRAM=39.2GiB | 93.1% done | ETA(epoch): 876s + [Epoch 279][30/66] loss=0.976464 avg=0.870287 VRAM=39.5GiB | 93.2% done | ETA(epoch): 687s + [Epoch 279][40/66] loss=0.875313 avg=0.866491 VRAM=39.2GiB | 93.2% done | ETA(epoch): 496s + [Epoch 279][50/66] loss=0.871015 avg=0.861419 VRAM=39.5GiB | 93.3% done | ETA(epoch): 305s + [Epoch 279][60/66] loss=0.912142 avg=0.854831 VRAM=39.2GiB | 93.3% done | ETA(epoch): 114s + Train loss: 0.854714 (1257.9s) ETA: 467min + Val loss: 0.821656 [t_0.0-0.2=1.0931 t_0.2-0.4=1.0098 t_0.4-0.6=0.7885 t_0.6-0.8=0.7148 t_0.8-1.0=0.6435] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 280/299 (93% done) --- + [Epoch 280][10/66] loss=0.805807 avg=0.825697 VRAM=39.5GiB | 93.4% done | ETA(epoch): 1066s + [Epoch 280][20/66] loss=0.782982 avg=0.834927 VRAM=39.2GiB | 93.4% done | ETA(epoch): 876s + [Epoch 280][30/66] loss=0.791933 avg=0.815974 VRAM=39.5GiB | 93.5% done | ETA(epoch): 688s + [Epoch 280][40/66] loss=0.803520 avg=0.822863 VRAM=39.2GiB | 93.5% done | ETA(epoch): 497s + [Epoch 280][50/66] loss=0.731078 avg=0.829800 VRAM=39.5GiB | 93.6% done | ETA(epoch): 306s + [Epoch 280][60/66] loss=0.795474 avg=0.833918 VRAM=39.2GiB | 93.6% done | ETA(epoch): 115s + Train loss: 0.841487 (1259.4s) ETA: 444min + Val loss: 0.821197 [t_0.0-0.2=1.0756 t_0.2-0.4=0.9603 t_0.4-0.6=0.8225 t_0.6-0.8=0.7116 t_0.8-1.0=0.6367] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 281/299 (94% done) --- + [Epoch 281][10/66] loss=0.839078 avg=0.835682 VRAM=39.5GiB | 93.7% done | ETA(epoch): 1066s + [Epoch 281][20/66] loss=0.854246 avg=0.848014 VRAM=39.2GiB | 93.8% done | ETA(epoch): 877s + [Epoch 281][30/66] loss=0.937383 avg=0.851435 VRAM=39.5GiB | 93.8% done | ETA(epoch): 686s + [Epoch 281][40/66] loss=0.872670 avg=0.852300 VRAM=39.2GiB | 93.9% done | ETA(epoch): 496s + [Epoch 281][50/66] loss=0.940279 avg=0.857844 VRAM=39.5GiB | 93.9% done | ETA(epoch): 305s + [Epoch 281][60/66] loss=0.881834 avg=0.862271 VRAM=39.2GiB | 94.0% done | ETA(epoch): 114s + Train loss: 0.862241 (1257.7s) ETA: 420min + Val loss: 0.853992 [t_0.0-0.2=1.0577 t_0.2-0.4=0.9708 t_0.4-0.6=0.8480 t_0.6-0.8=0.7052 t_0.8-1.0=0.6309] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0281 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 282/299 (94% done) --- + [Epoch 282][10/66] loss=0.920316 avg=0.879174 VRAM=39.5GiB | 94.1% done | ETA(epoch): 1065s + [Epoch 282][20/66] loss=0.840818 avg=0.837743 VRAM=39.2GiB | 94.1% done | ETA(epoch): 875s + [Epoch 282][30/66] loss=0.846221 avg=0.843868 VRAM=39.5GiB | 94.2% done | ETA(epoch): 685s + [Epoch 282][40/66] loss=0.719214 avg=0.835328 VRAM=39.2GiB | 94.2% done | ETA(epoch): 495s + [Epoch 282][50/66] loss=0.895971 avg=0.835892 VRAM=39.5GiB | 94.3% done | ETA(epoch): 305s + [Epoch 282][60/66] loss=0.794922 avg=0.843824 VRAM=39.2GiB | 94.3% done | ETA(epoch): 114s + Train loss: 0.846625 (1256.2s) ETA: 397min + Val loss: 0.843284 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9693 t_0.4-0.6=0.8300 t_0.6-0.8=0.7116 t_0.8-1.0=0.6438] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 283/299 (94% done) --- + [Epoch 283][10/66] loss=0.827537 avg=0.893102 VRAM=39.5GiB | 94.4% done | ETA(epoch): 1068s + [Epoch 283][20/66] loss=0.958493 avg=0.876033 VRAM=39.2GiB | 94.4% done | ETA(epoch): 877s + [Epoch 283][30/66] loss=0.824024 avg=0.874839 VRAM=39.5GiB | 94.5% done | ETA(epoch): 686s + [Epoch 283][40/66] loss=0.760349 avg=0.870714 VRAM=39.2GiB | 94.5% done | ETA(epoch): 495s + [Epoch 283][50/66] loss=0.822936 avg=0.869513 VRAM=39.5GiB | 94.6% done | ETA(epoch): 305s + [Epoch 283][60/66] loss=0.872218 avg=0.862605 VRAM=39.2GiB | 94.6% done | ETA(epoch): 114s + Train loss: 0.860424 (1257.3s) ETA: 373min + Val loss: 0.881027 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9908 t_0.4-0.6=0.8453 t_0.6-0.8=0.7091 t_0.8-1.0=0.6253] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 284/299 (95% done) --- + [Epoch 284][10/66] loss=0.943741 avg=0.855426 VRAM=39.5GiB | 94.7% done | ETA(epoch): 1066s + [Epoch 284][20/66] loss=0.854569 avg=0.857359 VRAM=39.2GiB | 94.8% done | ETA(epoch): 876s + [Epoch 284][30/66] loss=0.904920 avg=0.839114 VRAM=39.5GiB | 94.8% done | ETA(epoch): 685s + [Epoch 284][40/66] loss=0.883293 avg=0.844292 VRAM=39.2GiB | 94.9% done | ETA(epoch): 495s + [Epoch 284][50/66] loss=0.892802 avg=0.857219 VRAM=39.5GiB | 94.9% done | ETA(epoch): 305s + [Epoch 284][60/66] loss=0.983404 avg=0.857881 VRAM=39.2GiB | 95.0% done | ETA(epoch): 114s + Train loss: 0.859865 (1259.1s) ETA: 350min + Val loss: 0.834272 [t_0.0-0.2=1.0820 t_0.2-0.4=0.9638 t_0.4-0.6=0.8466 t_0.6-0.8=0.7175 t_0.8-1.0=0.6400] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0284 + Deleted old checkpoint: checkpoint_epoch_0281 +[MEM @ epoch 284 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 285/299 (95% done) --- + [Epoch 285][10/66] loss=0.820620 avg=0.824115 VRAM=39.5GiB | 95.1% done | ETA(epoch): 1065s + [Epoch 285][20/66] loss=0.911333 avg=0.841997 VRAM=39.2GiB | 95.1% done | ETA(epoch): 875s + [Epoch 285][30/66] loss=0.890247 avg=0.838758 VRAM=39.5GiB | 95.2% done | ETA(epoch): 685s + [Epoch 285][40/66] loss=0.766612 avg=0.832264 VRAM=39.2GiB | 95.2% done | ETA(epoch): 495s + [Epoch 285][50/66] loss=0.746667 avg=0.835293 VRAM=39.5GiB | 95.3% done | ETA(epoch): 305s + [Epoch 285][60/66] loss=0.797980 avg=0.839422 VRAM=39.2GiB | 95.3% done | ETA(epoch): 114s + Train loss: 0.841788 (1259.3s) ETA: 327min + Val loss: 0.842960 [t_0.0-0.2=1.0876 t_0.2-0.4=1.0405 t_0.4-0.6=0.8744 t_0.6-0.8=0.6784 t_0.8-1.0=0.6470] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 286/299 (95% done) --- + [Epoch 286][10/66] loss=0.847807 avg=0.869856 VRAM=39.5GiB | 95.4% done | ETA(epoch): 1064s + [Epoch 286][20/66] loss=0.864459 avg=0.848416 VRAM=39.2GiB | 95.4% done | ETA(epoch): 875s + [Epoch 286][30/66] loss=0.835303 avg=0.850805 VRAM=39.5GiB | 95.5% done | ETA(epoch): 685s + [Epoch 286][40/66] loss=0.906984 avg=0.854368 VRAM=39.2GiB | 95.5% done | ETA(epoch): 495s + [Epoch 286][50/66] loss=0.852930 avg=0.857399 VRAM=39.5GiB | 95.6% done | ETA(epoch): 305s + [Epoch 286][60/66] loss=0.968423 avg=0.862608 VRAM=39.2GiB | 95.6% done | ETA(epoch): 114s + Train loss: 0.857515 (1256.0s) ETA: 303min + Val loss: 0.860961 [t_0.0-0.2=1.0858 t_0.2-0.4=1.0110 t_0.4-0.6=0.8335 t_0.6-0.8=0.6969 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 287/299 (96% done) --- + [Epoch 287][10/66] loss=0.805123 avg=0.831427 VRAM=39.5GiB | 95.7% done | ETA(epoch): 1065s + [Epoch 287][20/66] loss=0.780333 avg=0.830129 VRAM=39.2GiB | 95.8% done | ETA(epoch): 876s + [Epoch 287][30/66] loss=0.859898 avg=0.841422 VRAM=39.5GiB | 95.8% done | ETA(epoch): 685s + [Epoch 287][40/66] loss=0.778947 avg=0.838055 VRAM=39.2GiB | 95.9% done | ETA(epoch): 495s + [Epoch 287][50/66] loss=0.859599 avg=0.845991 VRAM=39.5GiB | 95.9% done | ETA(epoch): 305s + [Epoch 287][60/66] loss=0.829309 avg=0.851343 VRAM=39.2GiB | 96.0% done | ETA(epoch): 114s + Train loss: 0.853830 (1256.1s) ETA: 280min + Val loss: 0.872137 [t_0.0-0.2=1.0924 t_0.2-0.4=0.9805 t_0.4-0.6=0.8620 t_0.6-0.8=0.7109 t_0.8-1.0=0.6357] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 288/299 (96% done) --- + [Epoch 288][10/66] loss=0.951632 avg=0.870242 VRAM=39.5GiB | 96.1% done | ETA(epoch): 1065s + [Epoch 288][20/66] loss=0.866293 avg=0.871172 VRAM=39.2GiB | 96.1% done | ETA(epoch): 875s + [Epoch 288][30/66] loss=0.929100 avg=0.871213 VRAM=39.5GiB | 96.2% done | ETA(epoch): 685s + [Epoch 288][40/66] loss=0.808649 avg=0.868585 VRAM=39.2GiB | 96.2% done | ETA(epoch): 495s + [Epoch 288][50/66] loss=0.856467 avg=0.862965 VRAM=39.5GiB | 96.3% done | ETA(epoch): 304s + [Epoch 288][60/66] loss=0.912708 avg=0.861026 VRAM=39.2GiB | 96.3% done | ETA(epoch): 114s + Train loss: 0.858070 (1256.0s) ETA: 257min + Val loss: 0.853255 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9949 t_0.4-0.6=0.8517 t_0.6-0.8=0.7020 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 289/299 (96% done) --- + [Epoch 289][10/66] loss=0.826512 avg=0.839605 VRAM=39.5GiB | 96.4% done | ETA(epoch): 1067s + [Epoch 289][20/66] loss=0.816127 avg=0.863697 VRAM=39.2GiB | 96.4% done | ETA(epoch): 876s + [Epoch 289][30/66] loss=0.943129 avg=0.861061 VRAM=39.5GiB | 96.5% done | ETA(epoch): 685s + [Epoch 289][40/66] loss=0.867630 avg=0.868885 VRAM=39.2GiB | 96.5% done | ETA(epoch): 495s + [Epoch 289][50/66] loss=0.690919 avg=0.865092 VRAM=39.5GiB | 96.6% done | ETA(epoch): 305s + [Epoch 289][60/66] loss=0.851380 avg=0.860312 VRAM=39.2GiB | 96.6% done | ETA(epoch): 114s + Train loss: 0.859444 (1256.0s) ETA: 233min + Val loss: 0.907441 [t_0.0-0.2=1.0688 t_0.2-0.4=1.0230 t_0.4-0.6=0.8317 t_0.6-0.8=0.7398 t_0.8-1.0=0.6318] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 290/299 (97% done) --- + [Epoch 290][10/66] loss=0.837166 avg=0.860209 VRAM=39.5GiB | 96.7% done | ETA(epoch): 1075s + [Epoch 290][20/66] loss=0.705169 avg=0.850210 VRAM=39.2GiB | 96.8% done | ETA(epoch): 879s + [Epoch 290][30/66] loss=0.951516 avg=0.852280 VRAM=39.5GiB | 96.8% done | ETA(epoch): 687s + [Epoch 290][40/66] loss=0.816151 avg=0.858166 VRAM=39.2GiB | 96.9% done | ETA(epoch): 496s + [Epoch 290][50/66] loss=0.854080 avg=0.864403 VRAM=39.5GiB | 96.9% done | ETA(epoch): 305s + [Epoch 290][60/66] loss=0.880776 avg=0.856651 VRAM=39.2GiB | 97.0% done | ETA(epoch): 114s + Train loss: 0.859797 (1258.0s) ETA: 210min + Val loss: 0.822625 [t_0.0-0.2=1.0565 t_0.2-0.4=0.9624 t_0.4-0.6=0.8707 t_0.6-0.8=0.7125 t_0.8-1.0=0.6271] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 291/299 (97% done) --- + [Epoch 291][10/66] loss=0.761673 avg=0.805657 VRAM=39.5GiB | 97.1% done | ETA(epoch): 1065s + [Epoch 291][20/66] loss=0.814043 avg=0.835376 VRAM=39.2GiB | 97.1% done | ETA(epoch): 876s + [Epoch 291][30/66] loss=0.901555 avg=0.856868 VRAM=39.5GiB | 97.2% done | ETA(epoch): 685s + [Epoch 291][40/66] loss=0.747853 avg=0.850151 VRAM=39.2GiB | 97.2% done | ETA(epoch): 495s + [Epoch 291][50/66] loss=0.808206 avg=0.846127 VRAM=39.5GiB | 97.3% done | ETA(epoch): 305s + [Epoch 291][60/66] loss=1.031567 avg=0.849642 VRAM=39.2GiB | 97.3% done | ETA(epoch): 114s + Train loss: 0.850791 (1257.7s) ETA: 187min + Val loss: 0.865017 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9665 t_0.4-0.6=0.8405 t_0.6-0.8=0.7119 t_0.8-1.0=0.6297] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 292/299 (97% done) --- + [Epoch 292][10/66] loss=0.905460 avg=0.858371 VRAM=39.5GiB | 97.4% done | ETA(epoch): 1066s + [Epoch 292][20/66] loss=0.803822 avg=0.869909 VRAM=39.2GiB | 97.4% done | ETA(epoch): 875s + [Epoch 292][30/66] loss=0.942062 avg=0.874143 VRAM=39.5GiB | 97.5% done | ETA(epoch): 685s + [Epoch 292][40/66] loss=0.932627 avg=0.867051 VRAM=39.2GiB | 97.5% done | ETA(epoch): 495s + [Epoch 292][50/66] loss=0.873352 avg=0.859609 VRAM=39.5GiB | 97.6% done | ETA(epoch): 305s + [Epoch 292][60/66] loss=0.778830 avg=0.853301 VRAM=39.2GiB | 97.6% done | ETA(epoch): 114s + Train loss: 0.845906 (1258.7s) ETA: 163min + Val loss: 0.882742 [t_0.0-0.2=1.0806 t_0.2-0.4=1.0059 t_0.4-0.6=0.8495 t_0.6-0.8=0.7113 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 293/299 (98% done) --- + [Epoch 293][10/66] loss=0.772345 avg=0.859648 VRAM=39.5GiB | 97.7% done | ETA(epoch): 1066s + [Epoch 293][20/66] loss=0.772096 avg=0.846910 VRAM=39.2GiB | 97.8% done | ETA(epoch): 876s + [Epoch 293][30/66] loss=0.890657 avg=0.844260 VRAM=39.5GiB | 97.8% done | ETA(epoch): 685s + [Epoch 293][40/66] loss=0.910441 avg=0.851950 VRAM=39.2GiB | 97.9% done | ETA(epoch): 495s + [Epoch 293][50/66] loss=0.874723 avg=0.853268 VRAM=39.5GiB | 97.9% done | ETA(epoch): 305s + [Epoch 293][60/66] loss=0.957338 avg=0.852996 VRAM=39.2GiB | 98.0% done | ETA(epoch): 114s + Train loss: 0.852474 (1258.1s) ETA: 140min + Val loss: 0.852720 [t_0.0-0.2=1.0563 t_0.2-0.4=1.0094 t_0.4-0.6=0.8460 t_0.6-0.8=0.6762 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 294/299 (98% done) --- + [Epoch 294][10/66] loss=0.850974 avg=0.877043 VRAM=39.5GiB | 98.1% done | ETA(epoch): 1065s + [Epoch 294][20/66] loss=0.776485 avg=0.838472 VRAM=39.2GiB | 98.1% done | ETA(epoch): 875s + [Epoch 294][30/66] loss=0.792546 avg=0.827616 VRAM=39.5GiB | 98.2% done | ETA(epoch): 685s + [Epoch 294][40/66] loss=0.920720 avg=0.839199 VRAM=39.2GiB | 98.2% done | ETA(epoch): 495s + [Epoch 294][50/66] loss=0.803306 avg=0.837285 VRAM=39.5GiB | 98.3% done | ETA(epoch): 305s + [Epoch 294][60/66] loss=0.844642 avg=0.844010 VRAM=39.2GiB | 98.3% done | ETA(epoch): 114s + Train loss: 0.848553 (1256.4s) ETA: 117min + Val loss: 0.840138 [t_0.0-0.2=1.0700 t_0.2-0.4=1.0227 t_0.4-0.6=0.8439 t_0.6-0.8=0.7030 t_0.8-1.0=0.6367] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 295/299 (98% done) --- + [Epoch 295][10/66] loss=0.948205 avg=0.872424 VRAM=39.5GiB | 98.4% done | ETA(epoch): 1066s + [Epoch 295][20/66] loss=0.824731 avg=0.865010 VRAM=39.2GiB | 98.4% done | ETA(epoch): 876s + [Epoch 295][30/66] loss=0.911664 avg=0.872077 VRAM=39.5GiB | 98.5% done | ETA(epoch): 686s + [Epoch 295][40/66] loss=0.828633 avg=0.859569 VRAM=39.2GiB | 98.5% done | ETA(epoch): 495s + [Epoch 295][50/66] loss=0.728735 avg=0.851716 VRAM=39.5GiB | 98.6% done | ETA(epoch): 305s + [Epoch 295][60/66] loss=0.951128 avg=0.852653 VRAM=39.2GiB | 98.6% done | ETA(epoch): 114s + Train loss: 0.849633 (1256.8s) ETA: 93min + Val loss: 0.854993 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0046 t_0.4-0.6=0.8085 t_0.6-0.8=0.6988 t_0.8-1.0=0.6363] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 296/299 (99% done) --- + [Epoch 296][10/66] loss=0.887845 avg=0.857816 VRAM=39.5GiB | 98.7% done | ETA(epoch): 1065s + [Epoch 296][20/66] loss=0.792378 avg=0.847787 VRAM=39.2GiB | 98.8% done | ETA(epoch): 875s + [Epoch 296][30/66] loss=0.677775 avg=0.840911 VRAM=39.5GiB | 98.8% done | ETA(epoch): 685s + [Epoch 296][40/66] loss=0.826628 avg=0.843610 VRAM=39.2GiB | 98.9% done | ETA(epoch): 495s + [Epoch 296][50/66] loss=0.890438 avg=0.840377 VRAM=39.5GiB | 98.9% done | ETA(epoch): 304s + [Epoch 296][60/66] loss=0.854710 avg=0.838002 VRAM=39.2GiB | 99.0% done | ETA(epoch): 114s + Train loss: 0.839006 (1255.9s) ETA: 70min + Val loss: 0.867376 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9790 t_0.4-0.6=0.8167 t_0.6-0.8=0.7113 t_0.8-1.0=0.6354] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 297/299 (99% done) --- + [Epoch 297][10/66] loss=0.834793 avg=0.845103 VRAM=39.5GiB | 99.1% done | ETA(epoch): 1065s + [Epoch 297][20/66] loss=0.914324 avg=0.849661 VRAM=39.2GiB | 99.1% done | ETA(epoch): 880s + [Epoch 297][30/66] loss=0.856722 avg=0.848701 VRAM=39.5GiB | 99.2% done | ETA(epoch): 687s + [Epoch 297][40/66] loss=0.943702 avg=0.862960 VRAM=39.2GiB | 99.2% done | ETA(epoch): 496s + [Epoch 297][50/66] loss=0.774650 avg=0.860431 VRAM=39.5GiB | 99.3% done | ETA(epoch): 305s + [Epoch 297][60/66] loss=0.918752 avg=0.866422 VRAM=39.2GiB | 99.3% done | ETA(epoch): 114s + Train loss: 0.870593 (1258.4s) ETA: 47min + Val loss: 0.855463 [t_0.0-0.2=1.0796 t_0.2-0.4=1.0039 t_0.4-0.6=0.8316 t_0.6-0.8=0.7087 t_0.8-1.0=0.6141] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 298/299 (99% done) --- + [Epoch 298][10/66] loss=0.771813 avg=0.803982 VRAM=39.5GiB | 99.4% done | ETA(epoch): 1067s + [Epoch 298][20/66] loss=0.784375 avg=0.829010 VRAM=39.2GiB | 99.4% done | ETA(epoch): 876s + [Epoch 298][30/66] loss=0.983054 avg=0.848115 VRAM=39.5GiB | 99.5% done | ETA(epoch): 685s + [Epoch 298][40/66] loss=0.850603 avg=0.851912 VRAM=39.2GiB | 99.5% done | ETA(epoch): 495s + [Epoch 298][50/66] loss=0.896170 avg=0.856492 VRAM=39.5GiB | 99.6% done | ETA(epoch): 305s + [Epoch 298][60/66] loss=0.918099 avg=0.856387 VRAM=39.2GiB | 99.6% done | ETA(epoch): 114s + Train loss: 0.854973 (1256.7s) ETA: 23min + Val loss: 0.888461 [t_0.0-0.2=1.0825 t_0.2-0.4=0.9886 t_0.4-0.6=0.8534 t_0.6-0.8=0.7089 t_0.8-1.0=0.6288] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 299/299 (100% done) --- + [Epoch 299][10/66] loss=0.883690 avg=0.851944 VRAM=39.5GiB | 99.7% done | ETA(epoch): 1077s + [Epoch 299][20/66] loss=0.857310 avg=0.845561 VRAM=39.2GiB | 99.8% done | ETA(epoch): 880s + [Epoch 299][30/66] loss=0.853624 avg=0.844790 VRAM=39.5GiB | 99.8% done | ETA(epoch): 687s + [Epoch 299][40/66] loss=0.900251 avg=0.856226 VRAM=39.2GiB | 99.9% done | ETA(epoch): 496s + [Epoch 299][50/66] loss=0.860915 avg=0.859118 VRAM=39.5GiB | 99.9% done | ETA(epoch): 305s + [Epoch 299][60/66] loss=0.898715 avg=0.856705 VRAM=39.2GiB | 100.0% done | ETA(epoch): 114s + [MilestoneVis] train_0 step 19799 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 19799 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 19799 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 19799 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 19799 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 19799 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 19799 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 19799 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 19799 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 19799 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H100_a/milestone_vis/milestone_grid_test_5.png + Train loss: 0.854232 (2290.9s) ETA: 0min + Val loss: 0.862518 [t_0.0-0.2=1.0628 t_0.2-0.4=0.9648 t_0.4-0.6=0.8538 t_0.6-0.8=0.7195 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_512_dim_512_H100_a/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 22.7/188.5 GiB (12.0%) | VRAM: 38.8/93.1 GiB (41.7%) + [LossCurve WARN] could not save loss curve: No module named 'matplotlib' + +====================================================================== +Training complete. +====================================================================== +wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.4s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.4s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.4s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.4s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.4s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.9s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.9s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.9s) + m wandb: uploading wandb-summary.json 3.2KB/3.2KB (0.9s) + m wandb: +wandb: Run summary: +wandb: adapter_lr 1e-05 +wandb: best_val_loss 0.81232 +wandb: cpu_pct 0.7 +wandb: epoch 299 +wandb: final_train_loss 0.85423 +wandb: flops/batch_size 1 +wandb: flops/hdc2a 210436620288 +wandb: flops/total 274313924131460 +wandb: flops/transformer 274103487511172 +wandb: global_step 19793 +wandb: +10 ... +wandb: +wandb: View run lora_512_dim_512_H100_a-train-20260513-100116 at: https://wandb.ai/hku-xg-boost/lora_512_dim_512_H100_a/runs/ix8fn97c +wandb: View project at: https://wandb.ai/hku-xg-boost/lora_512_dim_512_H100_a +wandb: Synced 4 W&B file(s), 110 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260513_100116-ix8fn97c/logs diff --git a/lora_512_dim_768_H100_b/train.log b/lora_512_dim_768_H100_b/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ba231edfafe4da7fcbd05d32a14f8a51e14b6158 --- /dev/null +++ b/lora_512_dim_768_H100_b/train.log @@ -0,0 +1,4658 @@ + +====================================================================== +Command: train_script_ablation_b.py --name lora_512_dim_768_H100_b --batch-size 6 --adapter-lr 1e-3 --num-epochs 300 --lora-lr 1e-4 --adapter-mlp --lora-rank 512 --fusion-dim 768 --seed 42 +====================================================================== +Run dir : output/lora_512_dim_768_H100_b +Log file: output/lora_512_dim_768_H100_b/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run s8w3h8ai (0.3s) + m wandb: setting up run s8w3h8ai (0.3s) + m wandb: setting up run s8w3h8ai (0.3s) + m wandb: setting up run s8w3h8ai (0.3s) + m wandb: setting up run s8w3h8ai (0.3s) + m wandb: setting up run s8w3h8ai (0.8s) + m wandb: setting up run s8w3h8ai (0.8s) + m wandb: setting up run s8w3h8ai (0.8s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260505_152359-s8w3h8ai +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_512_dim_768_H100_b-train-20260505-152359 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_512_dim_768_H100_b +wandb: View run at https://wandb.ai/hku-xg-boost/lora_512_dim_768_H100_b/runs/s8w3h8ai + +Run Summary: + name lora_512_dim_768_H100_b + fusion MLP (lightweight) + LoRA ON (rank=512) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_512_dim_768_H100_b + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 300 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_512_dim_768_H100_b + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 12.6/188.5 GiB (6.7%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.7s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 13.0/188.5 GiB (6.9%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (60.5s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 15.2/188.5 GiB (8.1%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating Ablation Adapter (b: seg-only) +============================================================ + Ablation mode: (b) Segmentation-only condition +HDC²A: 4.3M params +Control: 0.0M params +Total trainable: 4.3M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=512, alpha=512.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 157.29M + +Parameter Statistics: + HDC²A Adapter: total=4.3M trainable=4.3M + ControlNet (frozen): total=4290.8M LoRA trainable=157.29M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 4.3M + LoRA 157.29M = 161.63M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 20 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~4800 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0361, std=0.5352 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.366504 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 45.83 GiB + [test 4/4] Gradient flow check... + HDC²A: 20/20 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.019409 + proj.weight: 0.015198 + semantic_encoder.proj.weight: 0.011292 + semantic_encoder.conv_stem.3.weight: 0.010864 + semantic_encoder.conv_stem.0.weight: 0.006134 + Test result: PASSED +[MEM @ after test] RAM: 15.8/188.5 GiB (8.4%) | VRAM: 38.2/93.1 GiB (41.1%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 362.104 GFLOPs + transformer: 274103.488 GFLOPs + total: 274465.591 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 300 epochs × 66 steps = 19800 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 1980, 3960, 5940, 7920, 9900, 11879, 13859, 15839, 17819, 19799] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/299 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/66] loss=0.876170 avg=0.909027 VRAM=39.5GiB | 0.1% done | ETA(epoch): 6719s + [Epoch 0][20/66] loss=1.011004 avg=0.926245 VRAM=39.2GiB | 0.1% done | ETA(epoch): 3200s + [Epoch 0][30/66] loss=0.908527 avg=0.918780 VRAM=39.5GiB | 0.2% done | ETA(epoch): 1899s + [Epoch 0][40/66] loss=0.879564 avg=0.924560 VRAM=39.2GiB | 0.2% done | ETA(epoch): 1155s + [Epoch 0][50/66] loss=0.917642 avg=0.924518 VRAM=39.5GiB | 0.3% done | ETA(epoch): 630s + [Epoch 0][60/66] loss=0.946092 avg=0.929279 VRAM=39.2GiB | 0.3% done | ETA(epoch): 216s + Train loss: 0.926652 (2276.1s) ETA: 11368min + Val loss: 0.963905 [t_0.0-0.2=1.0859 t_0.2-0.4=1.0173 t_0.4-0.6=0.8959 t_0.6-0.8=0.7738 t_0.8-1.0=0.8800] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 16.0/188.5 GiB (8.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 1/299 (0% done) --- + [Epoch 1][10/66] loss=0.945704 avg=0.866090 VRAM=39.5GiB | 0.4% done | ETA(epoch): 1072s + [Epoch 1][20/66] loss=1.005363 avg=0.887362 VRAM=39.2GiB | 0.4% done | ETA(epoch): 881s + [Epoch 1][30/66] loss=0.974210 avg=0.897721 VRAM=39.5GiB | 0.5% done | ETA(epoch): 694s + [Epoch 1][40/66] loss=0.995173 avg=0.902716 VRAM=39.2GiB | 0.5% done | ETA(epoch): 501s + [Epoch 1][50/66] loss=0.906814 avg=0.899347 VRAM=39.5GiB | 0.6% done | ETA(epoch): 308s + [Epoch 1][60/66] loss=0.879309 avg=0.902103 VRAM=39.2GiB | 0.6% done | ETA(epoch): 115s + Train loss: 0.902232 (1268.6s) ETA: 9049min + Val loss: 0.920365 [t_0.0-0.2=1.0825 t_0.2-0.4=1.0254 t_0.4-0.6=0.8833 t_0.6-0.8=0.7499 t_0.8-1.0=0.9440] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 16.1/188.5 GiB (8.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 2/299 (1% done) --- + [Epoch 2][10/66] loss=0.974576 avg=0.903603 VRAM=39.5GiB | 0.7% done | ETA(epoch): 1073s + [Epoch 2][20/66] loss=0.966227 avg=0.905596 VRAM=39.2GiB | 0.8% done | ETA(epoch): 881s + [Epoch 2][30/66] loss=0.957480 avg=0.909872 VRAM=39.5GiB | 0.8% done | ETA(epoch): 690s + [Epoch 2][40/66] loss=0.943856 avg=0.913481 VRAM=39.2GiB | 0.9% done | ETA(epoch): 498s + [Epoch 2][50/66] loss=0.813191 avg=0.907389 VRAM=39.5GiB | 0.9% done | ETA(epoch): 307s + [Epoch 2][60/66] loss=1.113677 avg=0.906972 VRAM=39.2GiB | 1.0% done | ETA(epoch): 115s + Train loss: 0.905945 (1264.4s) ETA: 8254min + Val loss: 0.946379 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9950 t_0.4-0.6=0.8786 t_0.6-0.8=0.7664 t_0.8-1.0=0.9650] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 3/299 (1% done) --- + [Epoch 3][10/66] loss=0.797060 avg=0.906130 VRAM=39.5GiB | 1.1% done | ETA(epoch): 1098s + [Epoch 3][20/66] loss=0.830950 avg=0.898348 VRAM=39.2GiB | 1.1% done | ETA(epoch): 892s + [Epoch 3][30/66] loss=0.779386 avg=0.886145 VRAM=39.5GiB | 1.2% done | ETA(epoch): 695s + [Epoch 3][40/66] loss=0.833084 avg=0.888339 VRAM=39.2GiB | 1.2% done | ETA(epoch): 501s + [Epoch 3][50/66] loss=0.974094 avg=0.887444 VRAM=39.5GiB | 1.3% done | ETA(epoch): 308s + [Epoch 3][60/66] loss=0.997477 avg=0.890351 VRAM=39.2GiB | 1.3% done | ETA(epoch): 115s + Train loss: 0.890927 (1268.7s) ETA: 7851min + Val loss: 0.898914 [t_0.0-0.2=1.0776 t_0.2-0.4=1.0324 t_0.4-0.6=0.8745 t_0.6-0.8=0.7488 t_0.8-1.0=0.8636] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 4/299 (1% done) --- + [Epoch 4][10/66] loss=0.855354 avg=0.853631 VRAM=39.5GiB | 1.4% done | ETA(epoch): 1073s + [Epoch 4][20/66] loss=0.836712 avg=0.858094 VRAM=39.2GiB | 1.4% done | ETA(epoch): 882s + [Epoch 4][30/66] loss=0.888435 avg=0.872576 VRAM=39.5GiB | 1.5% done | ETA(epoch): 690s + [Epoch 4][40/66] loss=0.776970 avg=0.872931 VRAM=39.2GiB | 1.5% done | ETA(epoch): 498s + [Epoch 4][50/66] loss=0.879164 avg=0.879187 VRAM=39.5GiB | 1.6% done | ETA(epoch): 307s + [Epoch 4][60/66] loss=0.888398 avg=0.882138 VRAM=39.2GiB | 1.6% done | ETA(epoch): 115s + Train loss: 0.879728 (1264.9s) ETA: 7596min + Val loss: 0.933048 [t_0.0-0.2=1.0842 t_0.2-0.4=1.0299 t_0.4-0.6=0.8368 t_0.6-0.8=0.7565 t_0.8-1.0=0.8152] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 5/299 (2% done) --- + [Epoch 5][10/66] loss=0.959235 avg=0.891934 VRAM=39.5GiB | 1.7% done | ETA(epoch): 1073s + [Epoch 5][20/66] loss=0.923146 avg=0.884418 VRAM=39.2GiB | 1.8% done | ETA(epoch): 883s + [Epoch 5][30/66] loss=0.945466 avg=0.888914 VRAM=39.5GiB | 1.8% done | ETA(epoch): 691s + [Epoch 5][40/66] loss=0.802395 avg=0.886707 VRAM=39.2GiB | 1.9% done | ETA(epoch): 499s + [Epoch 5][50/66] loss=0.920615 avg=0.884073 VRAM=39.5GiB | 1.9% done | ETA(epoch): 307s + [Epoch 5][60/66] loss=0.815357 avg=0.885003 VRAM=39.2GiB | 2.0% done | ETA(epoch): 115s + Train loss: 0.889949 (1265.2s) ETA: 7419min + Val loss: 0.910436 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9429 t_0.4-0.6=0.8776 t_0.6-0.8=0.7560 t_0.8-1.0=0.8726] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 6/299 (2% done) --- + [Epoch 6][10/66] loss=0.793460 avg=0.873318 VRAM=39.5GiB | 2.1% done | ETA(epoch): 1072s + [Epoch 6][20/66] loss=0.902787 avg=0.865698 VRAM=39.2GiB | 2.1% done | ETA(epoch): 881s + [Epoch 6][30/66] loss=0.823941 avg=0.875334 VRAM=39.5GiB | 2.2% done | ETA(epoch): 689s + [Epoch 6][40/66] loss=0.942075 avg=0.882038 VRAM=39.2GiB | 2.2% done | ETA(epoch): 498s + [Epoch 6][50/66] loss=0.828221 avg=0.884438 VRAM=39.5GiB | 2.3% done | ETA(epoch): 306s + [Epoch 6][60/66] loss=0.965114 avg=0.887111 VRAM=39.2GiB | 2.3% done | ETA(epoch): 115s + Train loss: 0.885618 (1264.3s) ETA: 7285min + Val loss: 0.935367 [t_0.0-0.2=1.0847 t_0.2-0.4=1.0137 t_0.4-0.6=0.8623 t_0.6-0.8=0.7447 t_0.8-1.0=0.8761] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0006 +[MEM @ epoch 6 end] RAM: 17.1/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 7/299 (2% done) --- + [Epoch 7][10/66] loss=0.907046 avg=0.884961 VRAM=39.5GiB | 2.4% done | ETA(epoch): 1072s + [Epoch 7][20/66] loss=0.942451 avg=0.900716 VRAM=39.2GiB | 2.4% done | ETA(epoch): 881s + [Epoch 7][30/66] loss=0.906150 avg=0.901651 VRAM=39.5GiB | 2.5% done | ETA(epoch): 689s + [Epoch 7][40/66] loss=1.078343 avg=0.899863 VRAM=39.2GiB | 2.5% done | ETA(epoch): 498s + [Epoch 7][50/66] loss=0.901217 avg=0.896734 VRAM=39.5GiB | 2.6% done | ETA(epoch): 307s + [Epoch 7][60/66] loss=1.043082 avg=0.894825 VRAM=39.2GiB | 2.6% done | ETA(epoch): 115s + Train loss: 0.895672 (1266.9s) ETA: 7181min + Val loss: 0.909191 [t_0.0-0.2=1.0745 t_0.2-0.4=1.0086 t_0.4-0.6=0.9252 t_0.6-0.8=0.7422 t_0.8-1.0=0.7744] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 8/299 (3% done) --- + [Epoch 8][10/66] loss=0.834037 avg=0.899583 VRAM=39.5GiB | 2.7% done | ETA(epoch): 1072s + [Epoch 8][20/66] loss=1.027024 avg=0.884420 VRAM=39.2GiB | 2.8% done | ETA(epoch): 881s + [Epoch 8][30/66] loss=0.864768 avg=0.888625 VRAM=39.5GiB | 2.8% done | ETA(epoch): 690s + [Epoch 8][40/66] loss=0.832216 avg=0.888309 VRAM=39.2GiB | 2.9% done | ETA(epoch): 498s + [Epoch 8][50/66] loss=0.864904 avg=0.890100 VRAM=39.5GiB | 2.9% done | ETA(epoch): 307s + [Epoch 8][60/66] loss=0.899209 avg=0.890222 VRAM=39.2GiB | 3.0% done | ETA(epoch): 115s + Train loss: 0.893044 (1264.9s) ETA: 7094min + Val loss: 0.907422 [t_0.0-0.2=1.1034 t_0.2-0.4=1.0328 t_0.4-0.6=0.8549 t_0.6-0.8=0.7260 t_0.8-1.0=0.7880] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 16.6/188.5 GiB (8.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 9/299 (3% done) --- + [Epoch 9][10/66] loss=0.874257 avg=0.874514 VRAM=39.5GiB | 3.1% done | ETA(epoch): 1071s + [Epoch 9][20/66] loss=0.855987 avg=0.875703 VRAM=39.2GiB | 3.1% done | ETA(epoch): 880s + [Epoch 9][30/66] loss=0.893427 avg=0.890494 VRAM=39.5GiB | 3.2% done | ETA(epoch): 690s + [Epoch 9][40/66] loss=0.860251 avg=0.888430 VRAM=39.2GiB | 3.2% done | ETA(epoch): 498s + [Epoch 9][50/66] loss=0.839187 avg=0.883738 VRAM=39.5GiB | 3.3% done | ETA(epoch): 307s + [Epoch 9][60/66] loss=0.842782 avg=0.887558 VRAM=39.2GiB | 3.3% done | ETA(epoch): 115s + Train loss: 0.890335 (1265.3s) ETA: 7020min + Val loss: 0.909851 [t_0.0-0.2=1.0723 t_0.2-0.4=1.0035 t_0.4-0.6=0.8721 t_0.6-0.8=0.7156 t_0.8-1.0=0.8364] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 10/299 (3% done) --- + [Epoch 10][10/66] loss=0.876193 avg=0.871069 VRAM=39.5GiB | 3.4% done | ETA(epoch): 1071s + [Epoch 10][20/66] loss=0.847033 avg=0.879089 VRAM=39.2GiB | 3.4% done | ETA(epoch): 887s + [Epoch 10][30/66] loss=0.932779 avg=0.893918 VRAM=39.5GiB | 3.5% done | ETA(epoch): 693s + [Epoch 10][40/66] loss=0.844167 avg=0.886746 VRAM=39.2GiB | 3.5% done | ETA(epoch): 500s + [Epoch 10][50/66] loss=0.740395 avg=0.884243 VRAM=39.5GiB | 3.6% done | ETA(epoch): 307s + [Epoch 10][60/66] loss=0.834569 avg=0.888471 VRAM=39.2GiB | 3.6% done | ETA(epoch): 115s + Train loss: 0.886932 (1267.0s) ETA: 6956min + Val loss: 0.904515 [t_0.0-0.2=1.0729 t_0.2-0.4=1.0012 t_0.4-0.6=0.8555 t_0.6-0.8=0.7260 t_0.8-1.0=0.7916] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0010 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 16.5/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 11/299 (4% done) --- + [Epoch 11][10/66] loss=0.829638 avg=0.859334 VRAM=39.5GiB | 3.7% done | ETA(epoch): 1073s + [Epoch 11][20/66] loss=0.933228 avg=0.886030 VRAM=39.2GiB | 3.8% done | ETA(epoch): 882s + [Epoch 11][30/66] loss=0.920164 avg=0.891516 VRAM=39.5GiB | 3.8% done | ETA(epoch): 690s + [Epoch 11][40/66] loss=0.826450 avg=0.887974 VRAM=39.2GiB | 3.9% done | ETA(epoch): 498s + [Epoch 11][50/66] loss=0.859059 avg=0.880831 VRAM=39.5GiB | 3.9% done | ETA(epoch): 307s + [Epoch 11][60/66] loss=0.893772 avg=0.877697 VRAM=39.2GiB | 4.0% done | ETA(epoch): 115s + Train loss: 0.878735 (1265.4s) ETA: 6898min + Val loss: 0.885637 [t_0.0-0.2=1.0672 t_0.2-0.4=1.0127 t_0.4-0.6=0.8627 t_0.6-0.8=0.7097 t_0.8-1.0=0.7777] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0011 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 16.6/188.5 GiB (8.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 12/299 (4% done) --- + [Epoch 12][10/66] loss=0.738025 avg=0.867948 VRAM=39.5GiB | 4.1% done | ETA(epoch): 1072s + [Epoch 12][20/66] loss=0.765743 avg=0.870322 VRAM=39.2GiB | 4.1% done | ETA(epoch): 881s + [Epoch 12][30/66] loss=1.003973 avg=0.889475 VRAM=39.5GiB | 4.2% done | ETA(epoch): 690s + [Epoch 12][40/66] loss=0.785733 avg=0.885372 VRAM=39.2GiB | 4.2% done | ETA(epoch): 498s + [Epoch 12][50/66] loss=0.839670 avg=0.886796 VRAM=39.5GiB | 4.3% done | ETA(epoch): 307s + [Epoch 12][60/66] loss=0.906032 avg=0.889611 VRAM=39.2GiB | 4.3% done | ETA(epoch): 115s + Train loss: 0.890252 (1265.1s) ETA: 6847min + Val loss: 0.873028 [t_0.0-0.2=1.0623 t_0.2-0.4=0.9901 t_0.4-0.6=0.8620 t_0.6-0.8=0.7598 t_0.8-1.0=0.7215] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0012 (BEST) + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 16.6/188.5 GiB (8.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 13/299 (4% done) --- + [Epoch 13][10/66] loss=0.873075 avg=0.898329 VRAM=39.5GiB | 4.4% done | ETA(epoch): 1073s + [Epoch 13][20/66] loss=0.935430 avg=0.886934 VRAM=39.2GiB | 4.4% done | ETA(epoch): 883s + [Epoch 13][30/66] loss=0.999855 avg=0.893128 VRAM=39.5GiB | 4.5% done | ETA(epoch): 691s + [Epoch 13][40/66] loss=0.963222 avg=0.897173 VRAM=39.2GiB | 4.5% done | ETA(epoch): 499s + [Epoch 13][50/66] loss=0.895790 avg=0.895536 VRAM=39.5GiB | 4.6% done | ETA(epoch): 307s + [Epoch 13][60/66] loss=0.875587 avg=0.893315 VRAM=39.2GiB | 4.6% done | ETA(epoch): 115s + Train loss: 0.889254 (1269.2s) ETA: 6800min + Val loss: 0.898638 [t_0.0-0.2=1.0720 t_0.2-0.4=1.0142 t_0.4-0.6=0.8549 t_0.6-0.8=0.7503 t_0.8-1.0=0.7513] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 16.6/188.5 GiB (8.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 14/299 (5% done) --- + [Epoch 14][10/66] loss=0.727186 avg=0.875803 VRAM=39.5GiB | 4.7% done | ETA(epoch): 1076s + [Epoch 14][20/66] loss=0.818478 avg=0.862839 VRAM=39.2GiB | 4.8% done | ETA(epoch): 883s + [Epoch 14][30/66] loss=0.770811 avg=0.856565 VRAM=39.5GiB | 4.8% done | ETA(epoch): 691s + [Epoch 14][40/66] loss=0.884628 avg=0.860321 VRAM=39.2GiB | 4.9% done | ETA(epoch): 499s + [Epoch 14][50/66] loss=0.815151 avg=0.862379 VRAM=39.5GiB | 4.9% done | ETA(epoch): 308s + [Epoch 14][60/66] loss=0.844746 avg=0.865417 VRAM=39.2GiB | 5.0% done | ETA(epoch): 115s + Train loss: 0.867744 (1268.8s) ETA: 6757min + Val loss: 0.863609 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9890 t_0.4-0.6=0.8723 t_0.6-0.8=0.7205 t_0.8-1.0=0.7170] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0014 (BEST) + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 15/299 (5% done) --- + [Epoch 15][10/66] loss=0.785932 avg=0.865449 VRAM=39.5GiB | 5.1% done | ETA(epoch): 1073s + [Epoch 15][20/66] loss=0.967081 avg=0.894188 VRAM=39.2GiB | 5.1% done | ETA(epoch): 882s + [Epoch 15][30/66] loss=0.860512 avg=0.900415 VRAM=39.5GiB | 5.2% done | ETA(epoch): 690s + [Epoch 15][40/66] loss=0.956862 avg=0.896563 VRAM=39.2GiB | 5.2% done | ETA(epoch): 500s + [Epoch 15][50/66] loss=0.871055 avg=0.894513 VRAM=39.5GiB | 5.3% done | ETA(epoch): 307s + [Epoch 15][60/66] loss=0.795354 avg=0.890641 VRAM=39.2GiB | 5.3% done | ETA(epoch): 115s + Train loss: 0.888892 (1267.3s) ETA: 6715min + Val loss: 0.887050 [t_0.0-0.2=1.0686 t_0.2-0.4=0.9818 t_0.4-0.6=0.8814 t_0.6-0.8=0.7501 t_0.8-1.0=0.7187] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 16.6/188.5 GiB (8.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 16/299 (5% done) --- + [Epoch 16][10/66] loss=0.880636 avg=0.882232 VRAM=39.5GiB | 5.4% done | ETA(epoch): 1071s + [Epoch 16][20/66] loss=1.012369 avg=0.887941 VRAM=39.2GiB | 5.4% done | ETA(epoch): 881s + [Epoch 16][30/66] loss=0.907389 avg=0.876450 VRAM=39.5GiB | 5.5% done | ETA(epoch): 692s + [Epoch 16][40/66] loss=0.880860 avg=0.875398 VRAM=39.2GiB | 5.5% done | ETA(epoch): 499s + [Epoch 16][50/66] loss=0.955974 avg=0.882195 VRAM=39.5GiB | 5.6% done | ETA(epoch): 307s + [Epoch 16][60/66] loss=0.958275 avg=0.884913 VRAM=39.2GiB | 5.6% done | ETA(epoch): 115s + Train loss: 0.881794 (1266.9s) ETA: 6676min + Val loss: 0.865866 [t_0.0-0.2=1.0859 t_0.2-0.4=0.9959 t_0.4-0.6=0.8491 t_0.6-0.8=0.7469 t_0.8-1.0=0.7092] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 17/299 (6% done) --- + [Epoch 17][10/66] loss=0.894780 avg=0.843466 VRAM=39.5GiB | 5.7% done | ETA(epoch): 1075s + [Epoch 17][20/66] loss=0.860253 avg=0.867482 VRAM=39.2GiB | 5.8% done | ETA(epoch): 882s + [Epoch 17][30/66] loss=0.929169 avg=0.870402 VRAM=39.5GiB | 5.8% done | ETA(epoch): 691s + [Epoch 17][40/66] loss=0.842917 avg=0.865196 VRAM=39.2GiB | 5.9% done | ETA(epoch): 499s + [Epoch 17][50/66] loss=0.962612 avg=0.871109 VRAM=39.5GiB | 5.9% done | ETA(epoch): 307s + [Epoch 17][60/66] loss=0.769013 avg=0.872947 VRAM=39.2GiB | 6.0% done | ETA(epoch): 115s + Train loss: 0.868250 (1266.3s) ETA: 6638min + Val loss: 0.859628 [t_0.0-0.2=1.0811 t_0.2-0.4=0.9976 t_0.4-0.6=0.8881 t_0.6-0.8=0.7053 t_0.8-1.0=0.7324] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0017 (BEST) + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 18/299 (6% done) --- + [Epoch 18][10/66] loss=0.868538 avg=0.853812 VRAM=39.5GiB | 6.1% done | ETA(epoch): 1073s + [Epoch 18][20/66] loss=0.881802 avg=0.859747 VRAM=39.2GiB | 6.1% done | ETA(epoch): 882s + [Epoch 18][30/66] loss=0.936929 avg=0.861434 VRAM=39.5GiB | 6.2% done | ETA(epoch): 690s + [Epoch 18][40/66] loss=0.788616 avg=0.858152 VRAM=39.2GiB | 6.2% done | ETA(epoch): 498s + [Epoch 18][50/66] loss=0.826480 avg=0.861186 VRAM=39.5GiB | 6.3% done | ETA(epoch): 307s + [Epoch 18][60/66] loss=0.938107 avg=0.861954 VRAM=39.2GiB | 6.3% done | ETA(epoch): 115s + Train loss: 0.864258 (1265.5s) ETA: 6601min + Val loss: 0.898689 [t_0.0-0.2=1.0788 t_0.2-0.4=1.0067 t_0.4-0.6=0.8745 t_0.6-0.8=0.7423 t_0.8-1.0=0.7125] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 19/299 (6% done) --- + [Epoch 19][10/66] loss=0.911569 avg=0.880869 VRAM=39.5GiB | 6.4% done | ETA(epoch): 1085s + [Epoch 19][20/66] loss=0.929582 avg=0.880054 VRAM=39.2GiB | 6.4% done | ETA(epoch): 887s + [Epoch 19][30/66] loss=1.001793 avg=0.895035 VRAM=39.5GiB | 6.5% done | ETA(epoch): 693s + [Epoch 19][40/66] loss=0.830696 avg=0.888711 VRAM=39.2GiB | 6.5% done | ETA(epoch): 500s + [Epoch 19][50/66] loss=0.818133 avg=0.886847 VRAM=39.5GiB | 6.6% done | ETA(epoch): 307s + [Epoch 19][60/66] loss=1.027379 avg=0.880311 VRAM=39.2GiB | 6.6% done | ETA(epoch): 115s + Train loss: 0.880129 (1267.5s) ETA: 6567min + Val loss: 0.873123 [t_0.0-0.2=1.0679 t_0.2-0.4=0.9980 t_0.4-0.6=0.8590 t_0.6-0.8=0.7339 t_0.8-1.0=0.7170] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 20/299 (7% done) --- + [Epoch 20][10/66] loss=0.840584 avg=0.860867 VRAM=39.5GiB | 6.7% done | ETA(epoch): 1073s + [Epoch 20][20/66] loss=0.797518 avg=0.848127 VRAM=39.2GiB | 6.8% done | ETA(epoch): 882s + [Epoch 20][30/66] loss=0.989051 avg=0.875278 VRAM=39.5GiB | 6.8% done | ETA(epoch): 690s + [Epoch 20][40/66] loss=0.834405 avg=0.877582 VRAM=39.2GiB | 6.9% done | ETA(epoch): 499s + [Epoch 20][50/66] loss=0.883346 avg=0.880279 VRAM=39.5GiB | 6.9% done | ETA(epoch): 307s + [Epoch 20][60/66] loss=0.923427 avg=0.886428 VRAM=39.2GiB | 7.0% done | ETA(epoch): 115s + Train loss: 0.884341 (1265.9s) ETA: 6533min + Val loss: 0.872190 [t_0.0-0.2=1.0869 t_0.2-0.4=1.0305 t_0.4-0.6=0.8558 t_0.6-0.8=0.7152 t_0.8-1.0=0.7086] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0020 +[MEM @ epoch 20 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 21/299 (7% done) --- + [Epoch 21][10/66] loss=0.809980 avg=0.888401 VRAM=39.5GiB | 7.1% done | ETA(epoch): 1073s + [Epoch 21][20/66] loss=0.915870 avg=0.895158 VRAM=39.2GiB | 7.1% done | ETA(epoch): 882s + [Epoch 21][30/66] loss=0.897870 avg=0.883151 VRAM=39.5GiB | 7.2% done | ETA(epoch): 691s + [Epoch 21][40/66] loss=0.816503 avg=0.876243 VRAM=39.2GiB | 7.2% done | ETA(epoch): 499s + [Epoch 21][50/66] loss=1.044523 avg=0.878307 VRAM=39.5GiB | 7.3% done | ETA(epoch): 307s + [Epoch 21][60/66] loss=0.933877 avg=0.878042 VRAM=39.2GiB | 7.3% done | ETA(epoch): 115s + Train loss: 0.877884 (1268.3s) ETA: 6501min + Val loss: 0.862435 [t_0.0-0.2=1.0875 t_0.2-0.4=0.9944 t_0.4-0.6=0.8305 t_0.6-0.8=0.7073 t_0.8-1.0=0.7129] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 22/299 (7% done) --- + [Epoch 22][10/66] loss=0.883966 avg=0.902268 VRAM=39.5GiB | 7.4% done | ETA(epoch): 1073s + [Epoch 22][20/66] loss=0.803291 avg=0.875708 VRAM=39.2GiB | 7.4% done | ETA(epoch): 881s + [Epoch 22][30/66] loss=0.832840 avg=0.871499 VRAM=39.5GiB | 7.5% done | ETA(epoch): 690s + [Epoch 22][40/66] loss=0.941917 avg=0.867919 VRAM=39.2GiB | 7.5% done | ETA(epoch): 498s + [Epoch 22][50/66] loss=0.840956 avg=0.863253 VRAM=39.5GiB | 7.6% done | ETA(epoch): 307s + [Epoch 22][60/66] loss=0.846455 avg=0.864243 VRAM=39.2GiB | 7.6% done | ETA(epoch): 115s + Train loss: 0.862433 (1265.1s) ETA: 6469min + Val loss: 0.870274 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0011 t_0.4-0.6=0.8685 t_0.6-0.8=0.7153 t_0.8-1.0=0.6951] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 23/299 (8% done) --- + [Epoch 23][10/66] loss=0.809254 avg=0.890937 VRAM=39.5GiB | 7.7% done | ETA(epoch): 1072s + [Epoch 23][20/66] loss=0.798997 avg=0.882470 VRAM=39.2GiB | 7.8% done | ETA(epoch): 882s + [Epoch 23][30/66] loss=0.921542 avg=0.879446 VRAM=39.5GiB | 7.8% done | ETA(epoch): 690s + [Epoch 23][40/66] loss=0.903509 avg=0.881351 VRAM=39.2GiB | 7.9% done | ETA(epoch): 498s + [Epoch 23][50/66] loss=0.919446 avg=0.880223 VRAM=39.5GiB | 7.9% done | ETA(epoch): 307s + [Epoch 23][60/66] loss=0.985472 avg=0.877568 VRAM=39.2GiB | 8.0% done | ETA(epoch): 115s + Train loss: 0.873193 (1265.6s) ETA: 6438min + Val loss: 0.878201 [t_0.0-0.2=1.0788 t_0.2-0.4=1.0182 t_0.4-0.6=0.8569 t_0.6-0.8=0.7404 t_0.8-1.0=0.6792] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 24/299 (8% done) --- + [Epoch 24][10/66] loss=0.955923 avg=0.835145 VRAM=39.5GiB | 8.1% done | ETA(epoch): 1072s + [Epoch 24][20/66] loss=0.917868 avg=0.855601 VRAM=39.2GiB | 8.1% done | ETA(epoch): 883s + [Epoch 24][30/66] loss=0.837553 avg=0.863693 VRAM=39.5GiB | 8.2% done | ETA(epoch): 700s + [Epoch 24][40/66] loss=0.851161 avg=0.857472 VRAM=39.2GiB | 8.2% done | ETA(epoch): 504s + [Epoch 24][50/66] loss=0.856270 avg=0.856450 VRAM=39.5GiB | 8.3% done | ETA(epoch): 309s + [Epoch 24][60/66] loss=0.838020 avg=0.857374 VRAM=39.2GiB | 8.3% done | ETA(epoch): 116s + Train loss: 0.858734 (1272.9s) ETA: 6409min + Val loss: 0.890896 [t_0.0-0.2=1.0798 t_0.2-0.4=0.9877 t_0.4-0.6=0.8362 t_0.6-0.8=0.7593 t_0.8-1.0=0.6995] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 25/299 (8% done) --- + [Epoch 25][10/66] loss=0.792767 avg=0.880543 VRAM=39.5GiB | 8.4% done | ETA(epoch): 1073s + [Epoch 25][20/66] loss=0.916311 avg=0.892585 VRAM=39.2GiB | 8.4% done | ETA(epoch): 881s + [Epoch 25][30/66] loss=0.826439 avg=0.882397 VRAM=39.5GiB | 8.5% done | ETA(epoch): 690s + [Epoch 25][40/66] loss=0.766880 avg=0.873683 VRAM=39.2GiB | 8.5% done | ETA(epoch): 499s + [Epoch 25][50/66] loss=0.885567 avg=0.872937 VRAM=39.5GiB | 8.6% done | ETA(epoch): 307s + [Epoch 25][60/66] loss=0.857492 avg=0.869282 VRAM=39.2GiB | 8.6% done | ETA(epoch): 115s + Train loss: 0.867366 (1265.8s) ETA: 6379min + Val loss: 0.870382 [t_0.0-0.2=1.0871 t_0.2-0.4=1.0252 t_0.4-0.6=0.9029 t_0.6-0.8=0.7163 t_0.8-1.0=0.6919] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 26/299 (9% done) --- + [Epoch 26][10/66] loss=0.862098 avg=0.870177 VRAM=39.5GiB | 8.7% done | ETA(epoch): 1073s + [Epoch 26][20/66] loss=0.838922 avg=0.863317 VRAM=39.2GiB | 8.8% done | ETA(epoch): 890s + [Epoch 26][30/66] loss=0.942926 avg=0.874044 VRAM=39.5GiB | 8.8% done | ETA(epoch): 694s + [Epoch 26][40/66] loss=0.901440 avg=0.870348 VRAM=39.2GiB | 8.9% done | ETA(epoch): 501s + [Epoch 26][50/66] loss=0.824010 avg=0.869721 VRAM=39.5GiB | 8.9% done | ETA(epoch): 308s + [Epoch 26][60/66] loss=0.969025 avg=0.870337 VRAM=39.2GiB | 9.0% done | ETA(epoch): 115s + Train loss: 0.871075 (1268.6s) ETA: 6350min + Val loss: 0.820789 [t_0.0-0.2=1.0798 t_0.2-0.4=1.0443 t_0.4-0.6=0.8553 t_0.6-0.8=0.7210 t_0.8-1.0=0.6839] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0026 (BEST) + Deleted old checkpoint: checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 16.5/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 27/299 (9% done) --- + [Epoch 27][10/66] loss=0.888084 avg=0.854108 VRAM=39.5GiB | 9.1% done | ETA(epoch): 1084s + [Epoch 27][20/66] loss=0.847680 avg=0.844418 VRAM=39.2GiB | 9.1% done | ETA(epoch): 886s + [Epoch 27][30/66] loss=0.929293 avg=0.846900 VRAM=39.5GiB | 9.2% done | ETA(epoch): 692s + [Epoch 27][40/66] loss=0.852919 avg=0.856384 VRAM=39.2GiB | 9.2% done | ETA(epoch): 500s + [Epoch 27][50/66] loss=0.801656 avg=0.860580 VRAM=39.5GiB | 9.3% done | ETA(epoch): 307s + [Epoch 27][60/66] loss=1.050356 avg=0.868049 VRAM=39.2GiB | 9.3% done | ETA(epoch): 115s + Train loss: 0.871070 (1269.2s) ETA: 6322min + Val loss: 0.879922 [t_0.0-0.2=1.0577 t_0.2-0.4=0.9832 t_0.4-0.6=0.8727 t_0.6-0.8=0.7324 t_0.8-1.0=0.7085] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 28/299 (9% done) --- + [Epoch 28][10/66] loss=0.912899 avg=0.886271 VRAM=39.5GiB | 9.4% done | ETA(epoch): 1073s + [Epoch 28][20/66] loss=0.870090 avg=0.884230 VRAM=39.2GiB | 9.4% done | ETA(epoch): 881s + [Epoch 28][30/66] loss=0.959482 avg=0.885492 VRAM=39.5GiB | 9.5% done | ETA(epoch): 689s + [Epoch 28][40/66] loss=0.954980 avg=0.875301 VRAM=39.2GiB | 9.5% done | ETA(epoch): 498s + [Epoch 28][50/66] loss=0.883445 avg=0.877665 VRAM=39.5GiB | 9.6% done | ETA(epoch): 306s + [Epoch 28][60/66] loss=0.884257 avg=0.877177 VRAM=39.2GiB | 9.6% done | ETA(epoch): 115s + Train loss: 0.876767 (1266.4s) ETA: 6293min + Val loss: 0.889115 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0221 t_0.4-0.6=0.8401 t_0.6-0.8=0.7433 t_0.8-1.0=0.7024] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 29/299 (10% done) --- + [Epoch 29][10/66] loss=0.940975 avg=0.880268 VRAM=39.5GiB | 9.7% done | ETA(epoch): 1073s + [Epoch 29][20/66] loss=0.873763 avg=0.888761 VRAM=39.2GiB | 9.8% done | ETA(epoch): 881s + [Epoch 29][30/66] loss=0.938635 avg=0.882083 VRAM=39.5GiB | 9.8% done | ETA(epoch): 690s + [Epoch 29][40/66] loss=0.807796 avg=0.882828 VRAM=39.2GiB | 9.9% done | ETA(epoch): 499s + [Epoch 29][50/66] loss=0.932475 avg=0.889888 VRAM=39.5GiB | 9.9% done | ETA(epoch): 308s + [Epoch 29][60/66] loss=0.879920 avg=0.892903 VRAM=39.2GiB | 10.0% done | ETA(epoch): 115s + Train loss: 0.887073 (1267.8s) ETA: 6265min + Val loss: 0.865427 [t_0.0-0.2=1.0818 t_0.2-0.4=0.9591 t_0.4-0.6=0.8522 t_0.6-0.8=0.7333 t_0.8-1.0=0.6843] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0029 +[MEM @ epoch 29 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 30/299 (10% done) --- + [MilestoneVis] train_0 step 1980 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 1980 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 1980 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 1980 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 1980 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 1980 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 1980 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 1980 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 1980 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 1980 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + [Epoch 30][10/66] loss=0.818436 avg=0.864279 VRAM=39.5GiB | 10.1% done | ETA(epoch): 6765s + [Epoch 30][20/66] loss=0.959653 avg=0.864928 VRAM=39.2GiB | 10.1% done | ETA(epoch): 3219s + [Epoch 30][30/66] loss=0.830210 avg=0.863641 VRAM=39.5GiB | 10.2% done | ETA(epoch): 1909s + [Epoch 30][40/66] loss=0.810851 avg=0.878486 VRAM=39.2GiB | 10.2% done | ETA(epoch): 1159s + [Epoch 30][50/66] loss=0.857545 avg=0.874147 VRAM=39.5GiB | 10.3% done | ETA(epoch): 632s + [Epoch 30][60/66] loss=0.929761 avg=0.886335 VRAM=39.2GiB | 10.3% done | ETA(epoch): 217s + Train loss: 0.878619 (2280.5s) ETA: 6384min + Val loss: 0.877496 [t_0.0-0.2=1.0489 t_0.2-0.4=1.0030 t_0.4-0.6=0.8602 t_0.6-0.8=0.7555 t_0.8-1.0=0.6958] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 17.1/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 31/299 (10% done) --- + [Epoch 31][10/66] loss=0.782333 avg=0.814828 VRAM=39.5GiB | 10.4% done | ETA(epoch): 1072s + [Epoch 31][20/66] loss=0.918019 avg=0.846280 VRAM=39.2GiB | 10.4% done | ETA(epoch): 881s + [Epoch 31][30/66] loss=1.020177 avg=0.856352 VRAM=39.5GiB | 10.5% done | ETA(epoch): 689s + [Epoch 31][40/66] loss=0.736836 avg=0.859229 VRAM=39.2GiB | 10.5% done | ETA(epoch): 498s + [Epoch 31][50/66] loss=0.738348 avg=0.854382 VRAM=39.5GiB | 10.6% done | ETA(epoch): 306s + [Epoch 31][60/66] loss=0.893617 avg=0.862708 VRAM=39.2GiB | 10.6% done | ETA(epoch): 115s + Train loss: 0.863116 (1264.0s) ETA: 6351min + Val loss: 0.901055 [t_0.0-0.2=1.0787 t_0.2-0.4=0.9931 t_0.4-0.6=0.8749 t_0.6-0.8=0.7251 t_0.8-1.0=0.6997] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 32/299 (11% done) --- + [Epoch 32][10/66] loss=0.884730 avg=0.871262 VRAM=39.5GiB | 10.7% done | ETA(epoch): 1072s + [Epoch 32][20/66] loss=0.874042 avg=0.866247 VRAM=39.2GiB | 10.8% done | ETA(epoch): 881s + [Epoch 32][30/66] loss=0.881803 avg=0.862034 VRAM=39.5GiB | 10.8% done | ETA(epoch): 690s + [Epoch 32][40/66] loss=1.024066 avg=0.866501 VRAM=39.2GiB | 10.9% done | ETA(epoch): 503s + [Epoch 32][50/66] loss=0.814265 avg=0.871853 VRAM=39.5GiB | 10.9% done | ETA(epoch): 309s + [Epoch 32][60/66] loss=0.967246 avg=0.874030 VRAM=39.2GiB | 11.0% done | ETA(epoch): 116s + Train loss: 0.872571 (1271.7s) ETA: 6320min + Val loss: 0.884747 [t_0.0-0.2=1.0812 t_0.2-0.4=0.9697 t_0.4-0.6=0.8752 t_0.6-0.8=0.7310 t_0.8-1.0=0.6922] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 17.1/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 33/299 (11% done) --- + [Epoch 33][10/66] loss=0.783946 avg=0.853892 VRAM=39.5GiB | 11.1% done | ETA(epoch): 1071s + [Epoch 33][20/66] loss=0.981084 avg=0.874977 VRAM=39.2GiB | 11.1% done | ETA(epoch): 881s + [Epoch 33][30/66] loss=0.827786 avg=0.876977 VRAM=39.5GiB | 11.2% done | ETA(epoch): 693s + [Epoch 33][40/66] loss=0.780333 avg=0.872311 VRAM=39.2GiB | 11.2% done | ETA(epoch): 500s + [Epoch 33][50/66] loss=0.895408 avg=0.866570 VRAM=39.5GiB | 11.3% done | ETA(epoch): 308s + [Epoch 33][60/66] loss=0.829886 avg=0.868377 VRAM=39.2GiB | 11.3% done | ETA(epoch): 115s + Train loss: 0.869963 (1267.5s) ETA: 6289min + Val loss: 0.847089 [t_0.0-0.2=1.0555 t_0.2-0.4=0.9974 t_0.4-0.6=0.8487 t_0.6-0.8=0.7285 t_0.8-1.0=0.7266] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 34/299 (11% done) --- + [Epoch 34][10/66] loss=0.833233 avg=0.819360 VRAM=39.5GiB | 11.4% done | ETA(epoch): 1071s + [Epoch 34][20/66] loss=0.817178 avg=0.857207 VRAM=39.2GiB | 11.4% done | ETA(epoch): 888s + [Epoch 34][30/66] loss=0.843354 avg=0.862372 VRAM=39.5GiB | 11.5% done | ETA(epoch): 693s + [Epoch 34][40/66] loss=0.938186 avg=0.868079 VRAM=39.2GiB | 11.5% done | ETA(epoch): 500s + [Epoch 34][50/66] loss=0.868490 avg=0.863428 VRAM=39.5GiB | 11.6% done | ETA(epoch): 307s + [Epoch 34][60/66] loss=0.946487 avg=0.868076 VRAM=39.2GiB | 11.6% done | ETA(epoch): 115s + Train loss: 0.865366 (1267.1s) ETA: 6258min + Val loss: 0.859893 [t_0.0-0.2=1.0669 t_0.2-0.4=1.0036 t_0.4-0.6=0.8357 t_0.6-0.8=0.7176 t_0.8-1.0=0.6886] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 35/299 (12% done) --- + [Epoch 35][10/66] loss=0.945761 avg=0.875888 VRAM=39.5GiB | 11.7% done | ETA(epoch): 1071s + [Epoch 35][20/66] loss=0.845616 avg=0.876606 VRAM=39.2GiB | 11.8% done | ETA(epoch): 880s + [Epoch 35][30/66] loss=0.720577 avg=0.876784 VRAM=39.5GiB | 11.8% done | ETA(epoch): 689s + [Epoch 35][40/66] loss=0.949621 avg=0.874083 VRAM=39.2GiB | 11.9% done | ETA(epoch): 497s + [Epoch 35][50/66] loss=0.791507 avg=0.872276 VRAM=39.5GiB | 11.9% done | ETA(epoch): 306s + [Epoch 35][60/66] loss=0.794986 avg=0.865834 VRAM=39.2GiB | 12.0% done | ETA(epoch): 115s + Train loss: 0.869722 (1263.3s) ETA: 6227min + Val loss: 0.859356 [t_0.0-0.2=1.0903 t_0.2-0.4=1.0096 t_0.4-0.6=0.8543 t_0.6-0.8=0.7338 t_0.8-1.0=0.7001] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 17.2/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 36/299 (12% done) --- + [Epoch 36][10/66] loss=0.927256 avg=0.889136 VRAM=39.5GiB | 12.1% done | ETA(epoch): 1096s + [Epoch 36][20/66] loss=0.803866 avg=0.872421 VRAM=39.2GiB | 12.1% done | ETA(epoch): 890s + [Epoch 36][30/66] loss=1.012260 avg=0.862968 VRAM=39.5GiB | 12.2% done | ETA(epoch): 694s + [Epoch 36][40/66] loss=0.925631 avg=0.876711 VRAM=39.2GiB | 12.2% done | ETA(epoch): 501s + [Epoch 36][50/66] loss=0.806058 avg=0.876759 VRAM=39.5GiB | 12.3% done | ETA(epoch): 308s + [Epoch 36][60/66] loss=0.767141 avg=0.875515 VRAM=39.2GiB | 12.3% done | ETA(epoch): 115s + Train loss: 0.871660 (1268.0s) ETA: 6197min + Val loss: 0.848634 [t_0.0-0.2=1.0623 t_0.2-0.4=1.0024 t_0.4-0.6=0.8077 t_0.6-0.8=0.7403 t_0.8-1.0=0.6914] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 37/299 (12% done) --- + [Epoch 37][10/66] loss=0.895013 avg=0.854694 VRAM=39.5GiB | 12.4% done | ETA(epoch): 1072s + [Epoch 37][20/66] loss=0.847205 avg=0.857794 VRAM=39.2GiB | 12.4% done | ETA(epoch): 880s + [Epoch 37][30/66] loss=0.769767 avg=0.855721 VRAM=39.5GiB | 12.5% done | ETA(epoch): 689s + [Epoch 37][40/66] loss=0.867346 avg=0.849622 VRAM=39.2GiB | 12.5% done | ETA(epoch): 498s + [Epoch 37][50/66] loss=0.937331 avg=0.849353 VRAM=39.5GiB | 12.6% done | ETA(epoch): 306s + [Epoch 37][60/66] loss=0.979516 avg=0.852746 VRAM=39.2GiB | 12.6% done | ETA(epoch): 115s + Train loss: 0.858456 (1265.7s) ETA: 6167min + Val loss: 0.848910 [t_0.0-0.2=1.0922 t_0.2-0.4=0.9081 t_0.4-0.6=0.9046 t_0.6-0.8=0.7276 t_0.8-1.0=0.6963] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 38/299 (13% done) --- + [Epoch 38][10/66] loss=0.835276 avg=0.819741 VRAM=39.5GiB | 12.7% done | ETA(epoch): 1071s + [Epoch 38][20/66] loss=0.908605 avg=0.847981 VRAM=39.2GiB | 12.8% done | ETA(epoch): 880s + [Epoch 38][30/66] loss=0.765052 avg=0.849104 VRAM=39.5GiB | 12.8% done | ETA(epoch): 689s + [Epoch 38][40/66] loss=0.976064 avg=0.855753 VRAM=39.2GiB | 12.9% done | ETA(epoch): 498s + [Epoch 38][50/66] loss=0.743827 avg=0.852992 VRAM=39.5GiB | 12.9% done | ETA(epoch): 306s + [Epoch 38][60/66] loss=0.908443 avg=0.853548 VRAM=39.2GiB | 13.0% done | ETA(epoch): 115s + Train loss: 0.854814 (1263.8s) ETA: 6138min + Val loss: 0.896044 [t_0.0-0.2=1.0800 t_0.2-0.4=0.9880 t_0.4-0.6=0.8614 t_0.6-0.8=0.7324 t_0.8-1.0=0.6991] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 39/299 (13% done) --- + [Epoch 39][10/66] loss=0.903724 avg=0.862262 VRAM=39.5GiB | 13.1% done | ETA(epoch): 1072s + [Epoch 39][20/66] loss=0.863958 avg=0.873210 VRAM=39.2GiB | 13.1% done | ETA(epoch): 881s + [Epoch 39][30/66] loss=0.801698 avg=0.857542 VRAM=39.5GiB | 13.2% done | ETA(epoch): 689s + [Epoch 39][40/66] loss=0.830214 avg=0.858529 VRAM=39.2GiB | 13.2% done | ETA(epoch): 498s + [Epoch 39][50/66] loss=0.840834 avg=0.860240 VRAM=39.5GiB | 13.3% done | ETA(epoch): 307s + [Epoch 39][60/66] loss=0.766459 avg=0.860756 VRAM=39.2GiB | 13.3% done | ETA(epoch): 115s + Train loss: 0.860630 (1266.1s) ETA: 6109min + Val loss: 0.864402 [t_0.0-0.2=1.0712 t_0.2-0.4=0.9916 t_0.4-0.6=0.8432 t_0.6-0.8=0.7466 t_0.8-1.0=0.6933] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 17.2/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 40/299 (13% done) --- + [Epoch 40][10/66] loss=0.825985 avg=0.873953 VRAM=39.5GiB | 13.4% done | ETA(epoch): 1071s + [Epoch 40][20/66] loss=0.851190 avg=0.886484 VRAM=39.2GiB | 13.4% done | ETA(epoch): 880s + [Epoch 40][30/66] loss=0.804944 avg=0.872135 VRAM=39.5GiB | 13.5% done | ETA(epoch): 688s + [Epoch 40][40/66] loss=0.920391 avg=0.875819 VRAM=39.2GiB | 13.5% done | ETA(epoch): 498s + [Epoch 40][50/66] loss=0.864900 avg=0.882064 VRAM=39.5GiB | 13.6% done | ETA(epoch): 307s + [Epoch 40][60/66] loss=0.787221 avg=0.882742 VRAM=39.2GiB | 13.6% done | ETA(epoch): 115s + Train loss: 0.881074 (1265.1s) ETA: 6080min + Val loss: 0.874644 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9941 t_0.4-0.6=0.8581 t_0.6-0.8=0.7242 t_0.8-1.0=0.7025] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 17.2/188.5 GiB (9.1%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 41/299 (14% done) --- + [Epoch 41][10/66] loss=0.961900 avg=0.896765 VRAM=39.5GiB | 13.7% done | ETA(epoch): 1072s + [Epoch 41][20/66] loss=0.915489 avg=0.882719 VRAM=39.2GiB | 13.8% done | ETA(epoch): 881s + [Epoch 41][30/66] loss=0.834843 avg=0.871490 VRAM=39.5GiB | 13.8% done | ETA(epoch): 694s + [Epoch 41][40/66] loss=0.929121 avg=0.874063 VRAM=39.2GiB | 13.9% done | ETA(epoch): 500s + [Epoch 41][50/66] loss=0.903817 avg=0.868122 VRAM=39.5GiB | 13.9% done | ETA(epoch): 308s + [Epoch 41][60/66] loss=0.886324 avg=0.863722 VRAM=39.2GiB | 14.0% done | ETA(epoch): 115s + Train loss: 0.863645 (1267.9s) ETA: 6052min + Val loss: 0.860547 [t_0.0-0.2=1.0781 t_0.2-0.4=1.0081 t_0.4-0.6=0.8260 t_0.6-0.8=0.7030 t_0.8-1.0=0.6845] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 42/299 (14% done) --- + [Epoch 42][10/66] loss=0.856501 avg=0.856066 VRAM=39.5GiB | 14.1% done | ETA(epoch): 1072s + [Epoch 42][20/66] loss=0.988108 avg=0.865972 VRAM=39.2GiB | 14.1% done | ETA(epoch): 881s + [Epoch 42][30/66] loss=0.901905 avg=0.872937 VRAM=39.5GiB | 14.2% done | ETA(epoch): 689s + [Epoch 42][40/66] loss=0.837972 avg=0.865566 VRAM=39.2GiB | 14.2% done | ETA(epoch): 498s + [Epoch 42][50/66] loss=0.794946 avg=0.857021 VRAM=39.5GiB | 14.3% done | ETA(epoch): 306s + [Epoch 42][60/66] loss=0.906762 avg=0.855252 VRAM=39.2GiB | 14.3% done | ETA(epoch): 115s + Train loss: 0.859481 (1263.9s) ETA: 6023min + Val loss: 0.847597 [t_0.0-0.2=1.0680 t_0.2-0.4=1.0173 t_0.4-0.6=0.8047 t_0.6-0.8=0.7418 t_0.8-1.0=0.6856] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 43/299 (14% done) --- + [Epoch 43][10/66] loss=0.963124 avg=0.867930 VRAM=39.5GiB | 14.4% done | ETA(epoch): 1084s + [Epoch 43][20/66] loss=0.888205 avg=0.877142 VRAM=39.2GiB | 14.4% done | ETA(epoch): 886s + [Epoch 43][30/66] loss=0.859259 avg=0.862583 VRAM=39.5GiB | 14.5% done | ETA(epoch): 692s + [Epoch 43][40/66] loss=0.823929 avg=0.859362 VRAM=39.2GiB | 14.5% done | ETA(epoch): 500s + [Epoch 43][50/66] loss=0.890746 avg=0.862762 VRAM=39.5GiB | 14.6% done | ETA(epoch): 307s + [Epoch 43][60/66] loss=0.959115 avg=0.865978 VRAM=39.2GiB | 14.6% done | ETA(epoch): 115s + Train loss: 0.862563 (1267.1s) ETA: 5996min + Val loss: 0.832596 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9954 t_0.4-0.6=0.8281 t_0.6-0.8=0.7196 t_0.8-1.0=0.6827] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 44/299 (15% done) --- + [Epoch 44][10/66] loss=0.821630 avg=0.871797 VRAM=39.5GiB | 14.7% done | ETA(epoch): 1072s + [Epoch 44][20/66] loss=0.800066 avg=0.864434 VRAM=39.2GiB | 14.8% done | ETA(epoch): 881s + [Epoch 44][30/66] loss=0.987551 avg=0.854954 VRAM=39.5GiB | 14.8% done | ETA(epoch): 690s + [Epoch 44][40/66] loss=1.042637 avg=0.858990 VRAM=39.2GiB | 14.9% done | ETA(epoch): 498s + [Epoch 44][50/66] loss=0.828401 avg=0.855856 VRAM=39.5GiB | 14.9% done | ETA(epoch): 307s + [Epoch 44][60/66] loss=0.906089 avg=0.858159 VRAM=39.2GiB | 15.0% done | ETA(epoch): 115s + Train loss: 0.857861 (1264.5s) ETA: 5968min + Val loss: 0.873490 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9935 t_0.4-0.6=0.8782 t_0.6-0.8=0.7058 t_0.8-1.0=0.6806] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 45/299 (15% done) --- + [Epoch 45][10/66] loss=0.820999 avg=0.860067 VRAM=39.5GiB | 15.1% done | ETA(epoch): 1074s + [Epoch 45][20/66] loss=0.863438 avg=0.862521 VRAM=39.2GiB | 15.1% done | ETA(epoch): 881s + [Epoch 45][30/66] loss=0.873306 avg=0.867445 VRAM=39.5GiB | 15.2% done | ETA(epoch): 690s + [Epoch 45][40/66] loss=0.852785 avg=0.863386 VRAM=39.2GiB | 15.2% done | ETA(epoch): 498s + [Epoch 45][50/66] loss=0.876829 avg=0.860771 VRAM=39.5GiB | 15.3% done | ETA(epoch): 307s + [Epoch 45][60/66] loss=0.764139 avg=0.860549 VRAM=39.2GiB | 15.3% done | ETA(epoch): 115s + Train loss: 0.859213 (1264.6s) ETA: 5940min + Val loss: 0.851664 [t_0.0-0.2=1.0670 t_0.2-0.4=0.9671 t_0.4-0.6=0.8386 t_0.6-0.8=0.7189 t_0.8-1.0=0.6636] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 46/299 (15% done) --- + [Epoch 46][10/66] loss=0.812957 avg=0.885432 VRAM=39.5GiB | 15.4% done | ETA(epoch): 1072s + [Epoch 46][20/66] loss=0.891248 avg=0.880029 VRAM=39.2GiB | 15.4% done | ETA(epoch): 881s + [Epoch 46][30/66] loss=0.959107 avg=0.883227 VRAM=39.5GiB | 15.5% done | ETA(epoch): 690s + [Epoch 46][40/66] loss=0.850389 avg=0.879111 VRAM=39.2GiB | 15.5% done | ETA(epoch): 498s + [Epoch 46][50/66] loss=0.848218 avg=0.880915 VRAM=39.5GiB | 15.6% done | ETA(epoch): 308s + [Epoch 46][60/66] loss=0.955214 avg=0.878784 VRAM=39.2GiB | 15.6% done | ETA(epoch): 115s + Train loss: 0.875448 (1268.1s) ETA: 5913min + Val loss: 0.880770 [t_0.0-0.2=1.0707 t_0.2-0.4=1.0049 t_0.4-0.6=0.8137 t_0.6-0.8=0.7621 t_0.8-1.0=0.6693] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 47/299 (16% done) --- + [Epoch 47][10/66] loss=0.837437 avg=0.848842 VRAM=39.5GiB | 15.7% done | ETA(epoch): 1072s + [Epoch 47][20/66] loss=0.780059 avg=0.834841 VRAM=39.2GiB | 15.8% done | ETA(epoch): 881s + [Epoch 47][30/66] loss=0.985693 avg=0.855531 VRAM=39.5GiB | 15.8% done | ETA(epoch): 690s + [Epoch 47][40/66] loss=0.834967 avg=0.859611 VRAM=39.2GiB | 15.9% done | ETA(epoch): 498s + [Epoch 47][50/66] loss=0.930010 avg=0.861512 VRAM=39.5GiB | 15.9% done | ETA(epoch): 307s + [Epoch 47][60/66] loss=0.877511 avg=0.860212 VRAM=39.2GiB | 16.0% done | ETA(epoch): 115s + Train loss: 0.864188 (1265.1s) ETA: 5886min + Val loss: 0.860719 [t_0.0-0.2=1.0615 t_0.2-0.4=1.0080 t_0.4-0.6=0.8551 t_0.6-0.8=0.7348 t_0.8-1.0=0.6842] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 48/299 (16% done) --- + [Epoch 48][10/66] loss=0.847050 avg=0.822761 VRAM=39.5GiB | 16.1% done | ETA(epoch): 1072s + [Epoch 48][20/66] loss=0.965631 avg=0.852203 VRAM=39.2GiB | 16.1% done | ETA(epoch): 881s + [Epoch 48][30/66] loss=0.833328 avg=0.848127 VRAM=39.5GiB | 16.2% done | ETA(epoch): 689s + [Epoch 48][40/66] loss=0.805858 avg=0.849395 VRAM=39.2GiB | 16.2% done | ETA(epoch): 499s + [Epoch 48][50/66] loss=0.736803 avg=0.849899 VRAM=39.5GiB | 16.3% done | ETA(epoch): 307s + [Epoch 48][60/66] loss=0.901190 avg=0.856757 VRAM=39.2GiB | 16.3% done | ETA(epoch): 115s + Train loss: 0.852160 (1266.7s) ETA: 5859min + Val loss: 0.872514 [t_0.0-0.2=1.0621 t_0.2-0.4=1.0143 t_0.4-0.6=0.8437 t_0.6-0.8=0.7326 t_0.8-1.0=0.7057] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 49/299 (16% done) --- + [Epoch 49][10/66] loss=0.897118 avg=0.878553 VRAM=39.5GiB | 16.4% done | ETA(epoch): 1077s + [Epoch 49][20/66] loss=0.799606 avg=0.892761 VRAM=39.2GiB | 16.4% done | ETA(epoch): 883s + [Epoch 49][30/66] loss=0.961326 avg=0.883078 VRAM=39.5GiB | 16.5% done | ETA(epoch): 693s + [Epoch 49][40/66] loss=0.865514 avg=0.878175 VRAM=39.2GiB | 16.5% done | ETA(epoch): 500s + [Epoch 49][50/66] loss=0.760079 avg=0.873307 VRAM=39.5GiB | 16.6% done | ETA(epoch): 307s + [Epoch 49][60/66] loss=0.871157 avg=0.870134 VRAM=39.2GiB | 16.6% done | ETA(epoch): 115s + Train loss: 0.870321 (1267.2s) ETA: 5833min + Val loss: 0.886236 [t_0.0-0.2=1.0704 t_0.2-0.4=0.9869 t_0.4-0.6=0.8777 t_0.6-0.8=0.7139 t_0.8-1.0=0.6852] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 17.4/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 50/299 (17% done) --- + [Epoch 50][10/66] loss=0.916403 avg=0.871524 VRAM=39.5GiB | 16.7% done | ETA(epoch): 1073s + [Epoch 50][20/66] loss=0.957461 avg=0.867545 VRAM=39.2GiB | 16.8% done | ETA(epoch): 881s + [Epoch 50][30/66] loss=0.958501 avg=0.873835 VRAM=39.5GiB | 16.8% done | ETA(epoch): 690s + [Epoch 50][40/66] loss=0.812048 avg=0.874914 VRAM=39.2GiB | 16.9% done | ETA(epoch): 498s + [Epoch 50][50/66] loss=0.795105 avg=0.871598 VRAM=39.5GiB | 16.9% done | ETA(epoch): 306s + [Epoch 50][60/66] loss=0.787435 avg=0.867016 VRAM=39.2GiB | 17.0% done | ETA(epoch): 115s + Train loss: 0.864496 (1264.9s) ETA: 5806min + Val loss: 0.885331 [t_0.0-0.2=1.0846 t_0.2-0.4=0.9817 t_0.4-0.6=0.8813 t_0.6-0.8=0.7317 t_0.8-1.0=0.6966] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 51/299 (17% done) --- + [Epoch 51][10/66] loss=0.724477 avg=0.865991 VRAM=39.5GiB | 17.1% done | ETA(epoch): 1083s + [Epoch 51][20/66] loss=0.767228 avg=0.853697 VRAM=39.2GiB | 17.1% done | ETA(epoch): 885s + [Epoch 51][30/66] loss=0.787335 avg=0.848019 VRAM=39.5GiB | 17.2% done | ETA(epoch): 692s + [Epoch 51][40/66] loss=0.844850 avg=0.853572 VRAM=39.2GiB | 17.2% done | ETA(epoch): 500s + [Epoch 51][50/66] loss=0.860834 avg=0.851400 VRAM=39.5GiB | 17.3% done | ETA(epoch): 307s + [Epoch 51][60/66] loss=0.897190 avg=0.849050 VRAM=39.2GiB | 17.3% done | ETA(epoch): 115s + Train loss: 0.850867 (1267.2s) ETA: 5780min + Val loss: 0.851983 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9780 t_0.4-0.6=0.8752 t_0.6-0.8=0.7008 t_0.8-1.0=0.6677] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 52/299 (17% done) --- + [Epoch 52][10/66] loss=0.970771 avg=0.842191 VRAM=39.5GiB | 17.4% done | ETA(epoch): 1073s + [Epoch 52][20/66] loss=0.945267 avg=0.840435 VRAM=39.2GiB | 17.4% done | ETA(epoch): 881s + [Epoch 52][30/66] loss=0.812119 avg=0.855154 VRAM=39.5GiB | 17.5% done | ETA(epoch): 690s + [Epoch 52][40/66] loss=0.855817 avg=0.848044 VRAM=39.2GiB | 17.5% done | ETA(epoch): 498s + [Epoch 52][50/66] loss=0.777629 avg=0.848017 VRAM=39.5GiB | 17.6% done | ETA(epoch): 307s + [Epoch 52][60/66] loss=0.802414 avg=0.847215 VRAM=39.2GiB | 17.6% done | ETA(epoch): 115s + Train loss: 0.840994 (1265.1s) ETA: 5754min + Val loss: 0.867677 [t_0.0-0.2=1.0898 t_0.2-0.4=0.9729 t_0.4-0.6=0.8539 t_0.6-0.8=0.7206 t_0.8-1.0=0.6852] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 53/299 (18% done) --- + [Epoch 53][10/66] loss=0.776992 avg=0.837461 VRAM=39.5GiB | 17.7% done | ETA(epoch): 1074s + [Epoch 53][20/66] loss=0.825734 avg=0.845780 VRAM=39.2GiB | 17.8% done | ETA(epoch): 882s + [Epoch 53][30/66] loss=0.874351 avg=0.854052 VRAM=39.5GiB | 17.8% done | ETA(epoch): 690s + [Epoch 53][40/66] loss=0.824476 avg=0.856627 VRAM=39.2GiB | 17.9% done | ETA(epoch): 498s + [Epoch 53][50/66] loss=0.836709 avg=0.853366 VRAM=39.5GiB | 17.9% done | ETA(epoch): 307s + [Epoch 53][60/66] loss=0.763896 avg=0.861904 VRAM=39.2GiB | 18.0% done | ETA(epoch): 115s + Train loss: 0.863658 (1268.5s) ETA: 5728min + Val loss: 0.869898 [t_0.0-0.2=1.0789 t_0.2-0.4=1.0009 t_0.4-0.6=0.8283 t_0.6-0.8=0.6921 t_0.8-1.0=0.6766] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 54/299 (18% done) --- + [Epoch 54][10/66] loss=0.831041 avg=0.832385 VRAM=39.5GiB | 18.1% done | ETA(epoch): 1073s + [Epoch 54][20/66] loss=0.807448 avg=0.844287 VRAM=39.2GiB | 18.1% done | ETA(epoch): 882s + [Epoch 54][30/66] loss=0.836120 avg=0.844157 VRAM=39.5GiB | 18.2% done | ETA(epoch): 690s + [Epoch 54][40/66] loss=0.956287 avg=0.841713 VRAM=39.2GiB | 18.2% done | ETA(epoch): 498s + [Epoch 54][50/66] loss=0.824351 avg=0.850106 VRAM=39.5GiB | 18.3% done | ETA(epoch): 308s + [Epoch 54][60/66] loss=0.921589 avg=0.853935 VRAM=39.2GiB | 18.3% done | ETA(epoch): 115s + Train loss: 0.855932 (1270.0s) ETA: 5702min + Val loss: 0.866182 [t_0.0-0.2=1.0822 t_0.2-0.4=0.9538 t_0.4-0.6=0.8237 t_0.6-0.8=0.7475 t_0.8-1.0=0.6639] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 55/299 (18% done) --- + [Epoch 55][10/66] loss=0.992859 avg=0.864333 VRAM=39.5GiB | 18.4% done | ETA(epoch): 1073s + [Epoch 55][20/66] loss=0.902745 avg=0.872249 VRAM=39.2GiB | 18.4% done | ETA(epoch): 881s + [Epoch 55][30/66] loss=0.860335 avg=0.867685 VRAM=39.5GiB | 18.5% done | ETA(epoch): 690s + [Epoch 55][40/66] loss=0.815574 avg=0.865440 VRAM=39.2GiB | 18.5% done | ETA(epoch): 498s + [Epoch 55][50/66] loss=0.880604 avg=0.868600 VRAM=39.5GiB | 18.6% done | ETA(epoch): 307s + [Epoch 55][60/66] loss=0.860079 avg=0.866929 VRAM=39.2GiB | 18.6% done | ETA(epoch): 115s + Train loss: 0.865090 (1265.5s) ETA: 5676min + Val loss: 0.844684 [t_0.0-0.2=1.0570 t_0.2-0.4=1.0435 t_0.4-0.6=0.8545 t_0.6-0.8=0.7283 t_0.8-1.0=0.6772] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 56/299 (19% done) --- + [Epoch 56][10/66] loss=0.786648 avg=0.884887 VRAM=39.5GiB | 18.7% done | ETA(epoch): 1073s + [Epoch 56][20/66] loss=0.868468 avg=0.855578 VRAM=39.2GiB | 18.8% done | ETA(epoch): 882s + [Epoch 56][30/66] loss=0.772298 avg=0.850315 VRAM=39.5GiB | 18.8% done | ETA(epoch): 692s + [Epoch 56][40/66] loss=0.826546 avg=0.854230 VRAM=39.2GiB | 18.9% done | ETA(epoch): 500s + [Epoch 56][50/66] loss=0.904376 avg=0.852143 VRAM=39.5GiB | 18.9% done | ETA(epoch): 307s + [Epoch 56][60/66] loss=0.880954 avg=0.845207 VRAM=39.2GiB | 19.0% done | ETA(epoch): 115s + Train loss: 0.844631 (1266.9s) ETA: 5650min + Val loss: 0.857905 [t_0.0-0.2=1.0824 t_0.2-0.4=0.9706 t_0.4-0.6=0.8428 t_0.6-0.8=0.7238 t_0.8-1.0=0.6788] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 57/299 (19% done) --- + [Epoch 57][10/66] loss=0.862620 avg=0.846590 VRAM=39.5GiB | 19.1% done | ETA(epoch): 1072s + [Epoch 57][20/66] loss=0.890966 avg=0.842680 VRAM=39.2GiB | 19.1% done | ETA(epoch): 881s + [Epoch 57][30/66] loss=0.798955 avg=0.847537 VRAM=39.5GiB | 19.2% done | ETA(epoch): 692s + [Epoch 57][40/66] loss=0.873206 avg=0.856281 VRAM=39.2GiB | 19.2% done | ETA(epoch): 499s + [Epoch 57][50/66] loss=0.866057 avg=0.860201 VRAM=39.5GiB | 19.3% done | ETA(epoch): 307s + [Epoch 57][60/66] loss=0.926132 avg=0.868487 VRAM=39.2GiB | 19.3% done | ETA(epoch): 115s + Train loss: 0.869062 (1266.7s) ETA: 5625min + Val loss: 0.841856 [t_0.0-0.2=1.0920 t_0.2-0.4=1.0064 t_0.4-0.6=0.8289 t_0.6-0.8=0.7151 t_0.8-1.0=0.6810] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 58/299 (19% done) --- + [Epoch 58][10/66] loss=0.792280 avg=0.836372 VRAM=39.5GiB | 19.4% done | ETA(epoch): 1072s + [Epoch 58][20/66] loss=0.770579 avg=0.835732 VRAM=39.2GiB | 19.4% done | ETA(epoch): 881s + [Epoch 58][30/66] loss=0.851089 avg=0.840927 VRAM=39.5GiB | 19.5% done | ETA(epoch): 690s + [Epoch 58][40/66] loss=0.813766 avg=0.839409 VRAM=39.2GiB | 19.5% done | ETA(epoch): 498s + [Epoch 58][50/66] loss=0.847001 avg=0.848414 VRAM=39.5GiB | 19.6% done | ETA(epoch): 307s + [Epoch 58][60/66] loss=0.847440 avg=0.856164 VRAM=39.2GiB | 19.6% done | ETA(epoch): 115s + Train loss: 0.861752 (1264.7s) ETA: 5599min + Val loss: 0.867992 [t_0.0-0.2=1.0753 t_0.2-0.4=0.9510 t_0.4-0.6=0.8572 t_0.6-0.8=0.7201 t_0.8-1.0=0.6805] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 59/299 (20% done) --- + [Epoch 59][10/66] loss=0.894760 avg=0.870341 VRAM=39.5GiB | 19.7% done | ETA(epoch): 1074s + [Epoch 59][20/66] loss=0.958010 avg=0.893645 VRAM=39.2GiB | 19.8% done | ETA(epoch): 882s + [Epoch 59][30/66] loss=0.814741 avg=0.879940 VRAM=39.5GiB | 19.8% done | ETA(epoch): 690s + [Epoch 59][40/66] loss=0.898849 avg=0.872695 VRAM=39.2GiB | 19.9% done | ETA(epoch): 498s + [Epoch 59][50/66] loss=0.888152 avg=0.865207 VRAM=39.5GiB | 19.9% done | ETA(epoch): 307s + [Epoch 59][60/66] loss=0.887914 avg=0.870604 VRAM=39.2GiB | 20.0% done | ETA(epoch): 115s + Train loss: 0.873941 (1265.0s) ETA: 5574min + Val loss: 0.870563 [t_0.0-0.2=1.0621 t_0.2-0.4=0.9783 t_0.4-0.6=0.8825 t_0.6-0.8=0.7466 t_0.8-1.0=0.7046] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 60/299 (20% done) --- + [MilestoneVis] train_0 step 3960 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3960 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3960 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3960 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3960 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3960 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3960 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3960 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3960 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3960 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + [Epoch 60][10/66] loss=0.800673 avg=0.856499 VRAM=39.5GiB | 20.1% done | ETA(epoch): 6770s + [Epoch 60][20/66] loss=0.845536 avg=0.895190 VRAM=39.2GiB | 20.1% done | ETA(epoch): 3221s + [Epoch 60][30/66] loss=0.877427 avg=0.887594 VRAM=39.5GiB | 20.2% done | ETA(epoch): 1911s + [Epoch 60][40/66] loss=0.802298 avg=0.881349 VRAM=39.2GiB | 20.2% done | ETA(epoch): 1160s + [Epoch 60][50/66] loss=0.913486 avg=0.875983 VRAM=39.5GiB | 20.3% done | ETA(epoch): 632s + [Epoch 60][60/66] loss=0.855886 avg=0.872915 VRAM=39.2GiB | 20.3% done | ETA(epoch): 217s + Train loss: 0.871324 (2282.0s) ETA: 5615min + Val loss: 0.887636 [t_0.0-0.2=1.0899 t_0.2-0.4=1.0089 t_0.4-0.6=0.8629 t_0.6-0.8=0.7327 t_0.8-1.0=0.6811] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 61/299 (20% done) --- + [Epoch 61][10/66] loss=0.818800 avg=0.853510 VRAM=39.5GiB | 20.4% done | ETA(epoch): 1073s + [Epoch 61][20/66] loss=0.834838 avg=0.875071 VRAM=39.2GiB | 20.4% done | ETA(epoch): 881s + [Epoch 61][30/66] loss=0.835872 avg=0.872054 VRAM=39.5GiB | 20.5% done | ETA(epoch): 690s + [Epoch 61][40/66] loss=0.883494 avg=0.876090 VRAM=39.2GiB | 20.5% done | ETA(epoch): 498s + [Epoch 61][50/66] loss=0.762570 avg=0.867085 VRAM=39.5GiB | 20.6% done | ETA(epoch): 307s + [Epoch 61][60/66] loss=0.904759 avg=0.864630 VRAM=39.2GiB | 20.6% done | ETA(epoch): 115s + Train loss: 0.867471 (1266.9s) ETA: 5588min + Val loss: 0.877727 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0069 t_0.4-0.6=0.8030 t_0.6-0.8=0.7301 t_0.8-1.0=0.6760] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 62/299 (21% done) --- + [Epoch 62][10/66] loss=0.958217 avg=0.855952 VRAM=39.5GiB | 20.7% done | ETA(epoch): 1071s + [Epoch 62][20/66] loss=0.827464 avg=0.851873 VRAM=39.2GiB | 20.8% done | ETA(epoch): 881s + [Epoch 62][30/66] loss=0.791684 avg=0.856740 VRAM=39.5GiB | 20.8% done | ETA(epoch): 690s + [Epoch 62][40/66] loss=0.813499 avg=0.851794 VRAM=39.2GiB | 20.9% done | ETA(epoch): 498s + [Epoch 62][50/66] loss=0.981062 avg=0.860754 VRAM=39.5GiB | 20.9% done | ETA(epoch): 307s + [Epoch 62][60/66] loss=0.994727 avg=0.866792 VRAM=39.2GiB | 21.0% done | ETA(epoch): 115s + Train loss: 0.867600 (1269.0s) ETA: 5562min + Val loss: 0.880112 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0070 t_0.4-0.6=0.8313 t_0.6-0.8=0.7175 t_0.8-1.0=0.6911] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 63/299 (21% done) --- + [Epoch 63][10/66] loss=0.873239 avg=0.885346 VRAM=39.5GiB | 21.1% done | ETA(epoch): 1073s + [Epoch 63][20/66] loss=0.783180 avg=0.878891 VRAM=39.2GiB | 21.1% done | ETA(epoch): 882s + [Epoch 63][30/66] loss=0.884005 avg=0.887164 VRAM=39.5GiB | 21.2% done | ETA(epoch): 690s + [Epoch 63][40/66] loss=0.824353 avg=0.883086 VRAM=39.2GiB | 21.2% done | ETA(epoch): 498s + [Epoch 63][50/66] loss=0.781260 avg=0.887975 VRAM=39.5GiB | 21.3% done | ETA(epoch): 307s + [Epoch 63][60/66] loss=0.910710 avg=0.892270 VRAM=39.2GiB | 21.3% done | ETA(epoch): 115s + Train loss: 0.890515 (1265.2s) ETA: 5535min + Val loss: 0.829831 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9653 t_0.4-0.6=0.8289 t_0.6-0.8=0.7397 t_0.8-1.0=0.6612] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 64/299 (21% done) --- + [Epoch 64][10/66] loss=0.835059 avg=0.875800 VRAM=39.5GiB | 21.4% done | ETA(epoch): 1075s + [Epoch 64][20/66] loss=0.979477 avg=0.872516 VRAM=39.2GiB | 21.4% done | ETA(epoch): 882s + [Epoch 64][30/66] loss=0.940459 avg=0.866946 VRAM=39.5GiB | 21.5% done | ETA(epoch): 691s + [Epoch 64][40/66] loss=0.991885 avg=0.859207 VRAM=39.2GiB | 21.5% done | ETA(epoch): 499s + [Epoch 64][50/66] loss=1.015852 avg=0.863283 VRAM=39.5GiB | 21.6% done | ETA(epoch): 307s + [Epoch 64][60/66] loss=0.846690 avg=0.859697 VRAM=39.2GiB | 21.6% done | ETA(epoch): 115s + Train loss: 0.864370 (1265.9s) ETA: 5509min + Val loss: 0.886917 [t_0.0-0.2=1.0744 t_0.2-0.4=0.9758 t_0.4-0.6=0.8626 t_0.6-0.8=0.7406 t_0.8-1.0=0.6885] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 65/299 (22% done) --- + [Epoch 65][10/66] loss=0.843013 avg=0.874510 VRAM=39.5GiB | 21.7% done | ETA(epoch): 1073s + [Epoch 65][20/66] loss=0.895966 avg=0.865013 VRAM=39.2GiB | 21.8% done | ETA(epoch): 882s + [Epoch 65][30/66] loss=0.866087 avg=0.873895 VRAM=39.5GiB | 21.8% done | ETA(epoch): 691s + [Epoch 65][40/66] loss=0.854178 avg=0.871136 VRAM=39.2GiB | 21.9% done | ETA(epoch): 499s + [Epoch 65][50/66] loss=0.728853 avg=0.867011 VRAM=39.5GiB | 21.9% done | ETA(epoch): 307s + [Epoch 65][60/66] loss=0.887848 avg=0.867287 VRAM=39.2GiB | 22.0% done | ETA(epoch): 115s + Train loss: 0.871329 (1266.2s) ETA: 5483min + Val loss: 0.853279 [t_0.0-0.2=1.0757 t_0.2-0.4=1.0041 t_0.4-0.6=0.8402 t_0.6-0.8=0.7265 t_0.8-1.0=0.6893] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 66/299 (22% done) --- + [Epoch 66][10/66] loss=0.761137 avg=0.865206 VRAM=39.5GiB | 22.1% done | ETA(epoch): 1072s + [Epoch 66][20/66] loss=0.854603 avg=0.868641 VRAM=39.2GiB | 22.1% done | ETA(epoch): 881s + [Epoch 66][30/66] loss=0.844683 avg=0.873155 VRAM=39.5GiB | 22.2% done | ETA(epoch): 690s + [Epoch 66][40/66] loss=0.784623 avg=0.870306 VRAM=39.2GiB | 22.2% done | ETA(epoch): 498s + [Epoch 66][50/66] loss=0.818874 avg=0.865266 VRAM=39.5GiB | 22.3% done | ETA(epoch): 307s + [Epoch 66][60/66] loss=0.944109 avg=0.859343 VRAM=39.2GiB | 22.3% done | ETA(epoch): 115s + Train loss: 0.851663 (1264.6s) ETA: 5457min + Val loss: 0.848995 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9989 t_0.4-0.6=0.8412 t_0.6-0.8=0.7052 t_0.8-1.0=0.6692] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 67/299 (22% done) --- + [Epoch 67][10/66] loss=0.816614 avg=0.842265 VRAM=39.5GiB | 22.4% done | ETA(epoch): 1073s + [Epoch 67][20/66] loss=0.881774 avg=0.855332 VRAM=39.2GiB | 22.4% done | ETA(epoch): 882s + [Epoch 67][30/66] loss=0.883592 avg=0.856577 VRAM=39.5GiB | 22.5% done | ETA(epoch): 690s + [Epoch 67][40/66] loss=0.863158 avg=0.865861 VRAM=39.2GiB | 22.5% done | ETA(epoch): 499s + [Epoch 67][50/66] loss=0.844930 avg=0.868107 VRAM=39.5GiB | 22.6% done | ETA(epoch): 307s + [Epoch 67][60/66] loss=0.910500 avg=0.866240 VRAM=39.2GiB | 22.6% done | ETA(epoch): 115s + Train loss: 0.863012 (1265.4s) ETA: 5431min + Val loss: 0.886932 [t_0.0-0.2=1.0818 t_0.2-0.4=1.0053 t_0.4-0.6=0.8322 t_0.6-0.8=0.7558 t_0.8-1.0=0.6530] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 68/299 (23% done) --- + [Epoch 68][10/66] loss=0.815795 avg=0.873991 VRAM=39.5GiB | 22.7% done | ETA(epoch): 1083s + [Epoch 68][20/66] loss=0.943510 avg=0.879553 VRAM=39.2GiB | 22.8% done | ETA(epoch): 886s + [Epoch 68][30/66] loss=0.927485 avg=0.868878 VRAM=39.5GiB | 22.8% done | ETA(epoch): 692s + [Epoch 68][40/66] loss=0.954566 avg=0.865675 VRAM=39.2GiB | 22.9% done | ETA(epoch): 499s + [Epoch 68][50/66] loss=0.947859 avg=0.865417 VRAM=39.5GiB | 22.9% done | ETA(epoch): 307s + [Epoch 68][60/66] loss=0.848374 avg=0.869697 VRAM=39.2GiB | 23.0% done | ETA(epoch): 115s + Train loss: 0.870009 (1267.6s) ETA: 5405min + Val loss: 0.842809 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9666 t_0.4-0.6=0.8338 t_0.6-0.8=0.7225 t_0.8-1.0=0.6903] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 69/299 (23% done) --- + [Epoch 69][10/66] loss=0.918867 avg=0.888757 VRAM=39.5GiB | 23.1% done | ETA(epoch): 1074s + [Epoch 69][20/66] loss=0.830409 avg=0.869569 VRAM=39.2GiB | 23.1% done | ETA(epoch): 882s + [Epoch 69][30/66] loss=0.804988 avg=0.865452 VRAM=39.5GiB | 23.2% done | ETA(epoch): 690s + [Epoch 69][40/66] loss=0.882071 avg=0.861729 VRAM=39.2GiB | 23.2% done | ETA(epoch): 499s + [Epoch 69][50/66] loss=0.895217 avg=0.857108 VRAM=39.5GiB | 23.3% done | ETA(epoch): 307s + [Epoch 69][60/66] loss=0.758213 avg=0.849887 VRAM=39.2GiB | 23.3% done | ETA(epoch): 115s + Train loss: 0.845924 (1266.7s) ETA: 5379min + Val loss: 0.867113 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9916 t_0.4-0.6=0.8597 t_0.6-0.8=0.7318 t_0.8-1.0=0.6582] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 18.2/188.5 GiB (9.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 70/299 (23% done) --- + [Epoch 70][10/66] loss=0.808483 avg=0.873530 VRAM=39.5GiB | 23.4% done | ETA(epoch): 1075s + [Epoch 70][20/66] loss=0.924519 avg=0.854194 VRAM=39.2GiB | 23.4% done | ETA(epoch): 882s + [Epoch 70][30/66] loss=0.916515 avg=0.852698 VRAM=39.5GiB | 23.5% done | ETA(epoch): 690s + [Epoch 70][40/66] loss=0.877033 avg=0.863139 VRAM=39.2GiB | 23.5% done | ETA(epoch): 498s + [Epoch 70][50/66] loss=0.818219 avg=0.861817 VRAM=39.5GiB | 23.6% done | ETA(epoch): 307s + [Epoch 70][60/66] loss=0.894069 avg=0.857898 VRAM=39.2GiB | 23.6% done | ETA(epoch): 115s + Train loss: 0.859077 (1265.5s) ETA: 5354min + Val loss: 0.826270 [t_0.0-0.2=1.0743 t_0.2-0.4=0.9743 t_0.4-0.6=0.8180 t_0.6-0.8=0.7286 t_0.8-1.0=0.6557] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 71/299 (24% done) --- + [Epoch 71][10/66] loss=0.752371 avg=0.848752 VRAM=39.5GiB | 23.7% done | ETA(epoch): 1074s + [Epoch 71][20/66] loss=0.789815 avg=0.851251 VRAM=39.2GiB | 23.8% done | ETA(epoch): 883s + [Epoch 71][30/66] loss=0.928962 avg=0.858514 VRAM=39.5GiB | 23.8% done | ETA(epoch): 690s + [Epoch 71][40/66] loss=0.757771 avg=0.850461 VRAM=39.2GiB | 23.9% done | ETA(epoch): 498s + [Epoch 71][50/66] loss=0.810269 avg=0.849939 VRAM=39.5GiB | 23.9% done | ETA(epoch): 308s + [Epoch 71][60/66] loss=0.933473 avg=0.849306 VRAM=39.2GiB | 24.0% done | ETA(epoch): 115s + Train loss: 0.851720 (1267.9s) ETA: 5328min + Val loss: 0.861984 [t_0.0-0.2=1.0875 t_0.2-0.4=0.9827 t_0.4-0.6=0.8389 t_0.6-0.8=0.7178 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 72/299 (24% done) --- + [Epoch 72][10/66] loss=0.931473 avg=0.813270 VRAM=39.5GiB | 24.1% done | ETA(epoch): 1072s + [Epoch 72][20/66] loss=0.837506 avg=0.839137 VRAM=39.2GiB | 24.1% done | ETA(epoch): 882s + [Epoch 72][30/66] loss=0.847661 avg=0.847173 VRAM=39.5GiB | 24.2% done | ETA(epoch): 690s + [Epoch 72][40/66] loss=0.848300 avg=0.857829 VRAM=39.2GiB | 24.2% done | ETA(epoch): 500s + [Epoch 72][50/66] loss=0.833045 avg=0.853461 VRAM=39.5GiB | 24.3% done | ETA(epoch): 307s + [Epoch 72][60/66] loss=0.982525 avg=0.850555 VRAM=39.2GiB | 24.3% done | ETA(epoch): 115s + Train loss: 0.849151 (1267.2s) ETA: 5303min + Val loss: 0.881294 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9940 t_0.4-0.6=0.8648 t_0.6-0.8=0.6971 t_0.8-1.0=0.6670] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 73/299 (24% done) --- + [Epoch 73][10/66] loss=0.820530 avg=0.852342 VRAM=39.5GiB | 24.4% done | ETA(epoch): 1074s + [Epoch 73][20/66] loss=0.774114 avg=0.875687 VRAM=39.2GiB | 24.4% done | ETA(epoch): 883s + [Epoch 73][30/66] loss=1.051111 avg=0.869617 VRAM=39.5GiB | 24.5% done | ETA(epoch): 691s + [Epoch 73][40/66] loss=0.961155 avg=0.852113 VRAM=39.2GiB | 24.5% done | ETA(epoch): 499s + [Epoch 73][50/66] loss=0.845283 avg=0.854661 VRAM=39.5GiB | 24.6% done | ETA(epoch): 307s + [Epoch 73][60/66] loss=0.912866 avg=0.862941 VRAM=39.2GiB | 24.6% done | ETA(epoch): 115s + Train loss: 0.860024 (1265.7s) ETA: 5277min + Val loss: 0.852304 [t_0.0-0.2=1.0566 t_0.2-0.4=0.9670 t_0.4-0.6=0.8749 t_0.6-0.8=0.7029 t_0.8-1.0=0.6739] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 74/299 (25% done) --- + [Epoch 74][10/66] loss=0.874854 avg=0.854231 VRAM=39.5GiB | 24.7% done | ETA(epoch): 1073s + [Epoch 74][20/66] loss=0.865963 avg=0.874289 VRAM=39.2GiB | 24.8% done | ETA(epoch): 881s + [Epoch 74][30/66] loss=0.738449 avg=0.859492 VRAM=39.5GiB | 24.8% done | ETA(epoch): 691s + [Epoch 74][40/66] loss=0.949814 avg=0.859536 VRAM=39.2GiB | 24.9% done | ETA(epoch): 499s + [Epoch 74][50/66] loss=0.875817 avg=0.864471 VRAM=39.5GiB | 24.9% done | ETA(epoch): 307s + [Epoch 74][60/66] loss=0.801407 avg=0.859994 VRAM=39.2GiB | 25.0% done | ETA(epoch): 115s + Train loss: 0.856972 (1265.1s) ETA: 5252min + Val loss: 0.869287 [t_0.0-0.2=1.0704 t_0.2-0.4=1.0265 t_0.4-0.6=0.8403 t_0.6-0.8=0.7266 t_0.8-1.0=0.6796] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 75/299 (25% done) --- + [Epoch 75][10/66] loss=0.842588 avg=0.880918 VRAM=39.5GiB | 25.1% done | ETA(epoch): 1072s + [Epoch 75][20/66] loss=0.846821 avg=0.855196 VRAM=39.2GiB | 25.1% done | ETA(epoch): 881s + [Epoch 75][30/66] loss=0.939516 avg=0.868788 VRAM=39.5GiB | 25.2% done | ETA(epoch): 690s + [Epoch 75][40/66] loss=0.820879 avg=0.862478 VRAM=39.2GiB | 25.2% done | ETA(epoch): 498s + [Epoch 75][50/66] loss=0.795747 avg=0.862020 VRAM=39.5GiB | 25.3% done | ETA(epoch): 307s + [Epoch 75][60/66] loss=0.973827 avg=0.864244 VRAM=39.2GiB | 25.3% done | ETA(epoch): 115s + Train loss: 0.864753 (1264.7s) ETA: 5226min + Val loss: 0.840652 [t_0.0-0.2=1.0754 t_0.2-0.4=0.9862 t_0.4-0.6=0.8377 t_0.6-0.8=0.7278 t_0.8-1.0=0.6744] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 76/299 (25% done) --- + [Epoch 76][10/66] loss=0.846583 avg=0.860103 VRAM=39.5GiB | 25.4% done | ETA(epoch): 1074s + [Epoch 76][20/66] loss=0.860193 avg=0.851315 VRAM=39.2GiB | 25.4% done | ETA(epoch): 882s + [Epoch 76][30/66] loss=0.784792 avg=0.856981 VRAM=39.5GiB | 25.5% done | ETA(epoch): 690s + [Epoch 76][40/66] loss=0.829594 avg=0.857218 VRAM=39.2GiB | 25.5% done | ETA(epoch): 498s + [Epoch 76][50/66] loss=0.893534 avg=0.859010 VRAM=39.5GiB | 25.6% done | ETA(epoch): 307s + [Epoch 76][60/66] loss=0.949741 avg=0.858444 VRAM=39.2GiB | 25.6% done | ETA(epoch): 115s + Train loss: 0.861387 (1265.5s) ETA: 5201min + Val loss: 0.865180 [t_0.0-0.2=1.0677 t_0.2-0.4=1.0313 t_0.4-0.6=0.8200 t_0.6-0.8=0.7387 t_0.8-1.0=0.6734] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 77/299 (26% done) --- + [Epoch 77][10/66] loss=0.848703 avg=0.856750 VRAM=39.5GiB | 25.7% done | ETA(epoch): 1073s + [Epoch 77][20/66] loss=0.898304 avg=0.851026 VRAM=39.2GiB | 25.8% done | ETA(epoch): 881s + [Epoch 77][30/66] loss=0.877159 avg=0.860905 VRAM=39.5GiB | 25.8% done | ETA(epoch): 689s + [Epoch 77][40/66] loss=0.773468 avg=0.864860 VRAM=39.2GiB | 25.9% done | ETA(epoch): 498s + [Epoch 77][50/66] loss=0.923917 avg=0.868303 VRAM=39.5GiB | 25.9% done | ETA(epoch): 307s + [Epoch 77][60/66] loss=0.751585 avg=0.868592 VRAM=39.2GiB | 26.0% done | ETA(epoch): 115s + Train loss: 0.866919 (1267.4s) ETA: 5176min + Val loss: 0.843433 [t_0.0-0.2=1.0606 t_0.2-0.4=0.9909 t_0.4-0.6=0.8427 t_0.6-0.8=0.7357 t_0.8-1.0=0.6689] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 78/299 (26% done) --- + [Epoch 78][10/66] loss=0.739268 avg=0.829849 VRAM=39.5GiB | 26.1% done | ETA(epoch): 1071s + [Epoch 78][20/66] loss=0.767060 avg=0.842042 VRAM=39.2GiB | 26.1% done | ETA(epoch): 880s + [Epoch 78][30/66] loss=0.864963 avg=0.862343 VRAM=39.5GiB | 26.2% done | ETA(epoch): 689s + [Epoch 78][40/66] loss=0.776638 avg=0.858750 VRAM=39.2GiB | 26.2% done | ETA(epoch): 498s + [Epoch 78][50/66] loss=0.819987 avg=0.858654 VRAM=39.5GiB | 26.3% done | ETA(epoch): 306s + [Epoch 78][60/66] loss=0.807265 avg=0.861246 VRAM=39.2GiB | 26.3% done | ETA(epoch): 115s + Train loss: 0.865614 (1263.2s) ETA: 5151min + Val loss: 0.827532 [t_0.0-0.2=1.0795 t_0.2-0.4=0.9971 t_0.4-0.6=0.7966 t_0.6-0.8=0.7339 t_0.8-1.0=0.6616] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 18.2/188.5 GiB (9.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 79/299 (26% done) --- + [Epoch 79][10/66] loss=0.925175 avg=0.884906 VRAM=39.5GiB | 26.4% done | ETA(epoch): 1073s + [Epoch 79][20/66] loss=0.777270 avg=0.878053 VRAM=39.2GiB | 26.4% done | ETA(epoch): 881s + [Epoch 79][30/66] loss=0.990347 avg=0.868503 VRAM=39.5GiB | 26.5% done | ETA(epoch): 690s + [Epoch 79][40/66] loss=0.812565 avg=0.860424 VRAM=39.2GiB | 26.5% done | ETA(epoch): 498s + [Epoch 79][50/66] loss=0.900371 avg=0.869228 VRAM=39.5GiB | 26.6% done | ETA(epoch): 307s + [Epoch 79][60/66] loss=0.784014 avg=0.868291 VRAM=39.2GiB | 26.6% done | ETA(epoch): 115s + Train loss: 0.863503 (1265.0s) ETA: 5126min + Val loss: 0.871176 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9665 t_0.4-0.6=0.8750 t_0.6-0.8=0.7601 t_0.8-1.0=0.6933] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 80/299 (27% done) --- + [Epoch 80][10/66] loss=0.945751 avg=0.883851 VRAM=39.5GiB | 26.7% done | ETA(epoch): 1074s + [Epoch 80][20/66] loss=0.954988 avg=0.874689 VRAM=39.2GiB | 26.8% done | ETA(epoch): 882s + [Epoch 80][30/66] loss=0.861765 avg=0.863697 VRAM=39.5GiB | 26.8% done | ETA(epoch): 690s + [Epoch 80][40/66] loss=0.929673 avg=0.861902 VRAM=39.2GiB | 26.9% done | ETA(epoch): 498s + [Epoch 80][50/66] loss=0.760431 avg=0.861138 VRAM=39.5GiB | 26.9% done | ETA(epoch): 307s + [Epoch 80][60/66] loss=1.056800 avg=0.866149 VRAM=39.2GiB | 27.0% done | ETA(epoch): 115s + Train loss: 0.866506 (1264.5s) ETA: 5101min + Val loss: 0.873502 [t_0.0-0.2=1.0624 t_0.2-0.4=0.9729 t_0.4-0.6=0.8712 t_0.6-0.8=0.7223 t_0.8-1.0=0.6789] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 81/299 (27% done) --- + [Epoch 81][10/66] loss=0.777326 avg=0.848132 VRAM=39.5GiB | 27.1% done | ETA(epoch): 1073s + [Epoch 81][20/66] loss=0.821013 avg=0.845994 VRAM=39.2GiB | 27.1% done | ETA(epoch): 881s + [Epoch 81][30/66] loss=0.909071 avg=0.845911 VRAM=39.5GiB | 27.2% done | ETA(epoch): 690s + [Epoch 81][40/66] loss=0.741860 avg=0.853805 VRAM=39.2GiB | 27.2% done | ETA(epoch): 498s + [Epoch 81][50/66] loss=0.785527 avg=0.850562 VRAM=39.5GiB | 27.3% done | ETA(epoch): 306s + [Epoch 81][60/66] loss=0.848135 avg=0.848439 VRAM=39.2GiB | 27.3% done | ETA(epoch): 115s + Train loss: 0.850861 (1264.4s) ETA: 5076min + Val loss: 0.894505 [t_0.0-0.2=1.0774 t_0.2-0.4=0.9910 t_0.4-0.6=0.8252 t_0.6-0.8=0.7266 t_0.8-1.0=0.6649] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 82/299 (27% done) --- + [Epoch 82][10/66] loss=0.825390 avg=0.870728 VRAM=39.5GiB | 27.4% done | ETA(epoch): 1072s + [Epoch 82][20/66] loss=0.840400 avg=0.852943 VRAM=39.2GiB | 27.4% done | ETA(epoch): 881s + [Epoch 82][30/66] loss=0.854991 avg=0.859064 VRAM=39.5GiB | 27.5% done | ETA(epoch): 690s + [Epoch 82][40/66] loss=0.887993 avg=0.859780 VRAM=39.2GiB | 27.5% done | ETA(epoch): 498s + [Epoch 82][50/66] loss=0.838071 avg=0.859256 VRAM=39.5GiB | 27.6% done | ETA(epoch): 307s + [Epoch 82][60/66] loss=0.699711 avg=0.852603 VRAM=39.2GiB | 27.6% done | ETA(epoch): 115s + Train loss: 0.854685 (1265.1s) ETA: 5051min + Val loss: 0.900889 [t_0.0-0.2=1.0739 t_0.2-0.4=1.0046 t_0.4-0.6=0.8286 t_0.6-0.8=0.7250 t_0.8-1.0=0.6662] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 83/299 (28% done) --- + [Epoch 83][10/66] loss=0.760621 avg=0.868611 VRAM=39.5GiB | 27.7% done | ETA(epoch): 1096s + [Epoch 83][20/66] loss=0.739450 avg=0.853337 VRAM=39.2GiB | 27.8% done | ETA(epoch): 891s + [Epoch 83][30/66] loss=0.894925 avg=0.856141 VRAM=39.5GiB | 27.8% done | ETA(epoch): 695s + [Epoch 83][40/66] loss=0.750541 avg=0.854621 VRAM=39.2GiB | 27.9% done | ETA(epoch): 501s + [Epoch 83][50/66] loss=0.895148 avg=0.854642 VRAM=39.5GiB | 27.9% done | ETA(epoch): 308s + [Epoch 83][60/66] loss=0.822534 avg=0.856932 VRAM=39.2GiB | 28.0% done | ETA(epoch): 115s + Train loss: 0.859146 (1269.2s) ETA: 5026min + Val loss: 0.872399 [t_0.0-0.2=1.0707 t_0.2-0.4=1.0404 t_0.4-0.6=0.7992 t_0.6-0.8=0.7272 t_0.8-1.0=0.6922] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 84/299 (28% done) --- + [Epoch 84][10/66] loss=0.832571 avg=0.825165 VRAM=39.5GiB | 28.1% done | ETA(epoch): 1072s + [Epoch 84][20/66] loss=0.808606 avg=0.831852 VRAM=39.2GiB | 28.1% done | ETA(epoch): 881s + [Epoch 84][30/66] loss=0.969196 avg=0.845347 VRAM=39.5GiB | 28.2% done | ETA(epoch): 690s + [Epoch 84][40/66] loss=0.757680 avg=0.849114 VRAM=39.2GiB | 28.2% done | ETA(epoch): 499s + [Epoch 84][50/66] loss=0.888647 avg=0.850698 VRAM=39.5GiB | 28.3% done | ETA(epoch): 307s + [Epoch 84][60/66] loss=0.744381 avg=0.849566 VRAM=39.2GiB | 28.3% done | ETA(epoch): 115s + Train loss: 0.848832 (1269.5s) ETA: 5001min + Val loss: 0.849790 [t_0.0-0.2=1.0761 t_0.2-0.4=1.0008 t_0.4-0.6=0.8583 t_0.6-0.8=0.7101 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 85/299 (28% done) --- + [Epoch 85][10/66] loss=0.932462 avg=0.863574 VRAM=39.5GiB | 28.4% done | ETA(epoch): 1072s + [Epoch 85][20/66] loss=0.852096 avg=0.850852 VRAM=39.2GiB | 28.4% done | ETA(epoch): 882s + [Epoch 85][30/66] loss=0.877129 avg=0.862229 VRAM=39.5GiB | 28.5% done | ETA(epoch): 690s + [Epoch 85][40/66] loss=0.738088 avg=0.863251 VRAM=39.2GiB | 28.5% done | ETA(epoch): 498s + [Epoch 85][50/66] loss=0.798238 avg=0.861366 VRAM=39.5GiB | 28.6% done | ETA(epoch): 307s + [Epoch 85][60/66] loss=0.901576 avg=0.860396 VRAM=39.2GiB | 28.6% done | ETA(epoch): 115s + Train loss: 0.862763 (1265.4s) ETA: 4977min + Val loss: 0.868103 [t_0.0-0.2=1.0730 t_0.2-0.4=1.0017 t_0.4-0.6=0.8668 t_0.6-0.8=0.7165 t_0.8-1.0=0.6680] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 86/299 (29% done) --- + [Epoch 86][10/66] loss=1.037537 avg=0.895018 VRAM=39.5GiB | 28.7% done | ETA(epoch): 1075s + [Epoch 86][20/66] loss=0.825529 avg=0.891110 VRAM=39.2GiB | 28.8% done | ETA(epoch): 883s + [Epoch 86][30/66] loss=0.886407 avg=0.872989 VRAM=39.5GiB | 28.8% done | ETA(epoch): 691s + [Epoch 86][40/66] loss=0.824269 avg=0.865588 VRAM=39.2GiB | 28.9% done | ETA(epoch): 499s + [Epoch 86][50/66] loss=0.797352 avg=0.862207 VRAM=39.5GiB | 28.9% done | ETA(epoch): 307s + [Epoch 86][60/66] loss=0.932607 avg=0.862421 VRAM=39.2GiB | 29.0% done | ETA(epoch): 115s + Train loss: 0.860518 (1265.9s) ETA: 4952min + Val loss: 0.830967 [t_0.0-0.2=1.0824 t_0.2-0.4=0.9765 t_0.4-0.6=0.8729 t_0.6-0.8=0.7242 t_0.8-1.0=0.6626] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 87/299 (29% done) --- + [Epoch 87][10/66] loss=0.996624 avg=0.842245 VRAM=39.5GiB | 29.1% done | ETA(epoch): 1075s + [Epoch 87][20/66] loss=0.805822 avg=0.844292 VRAM=39.2GiB | 29.1% done | ETA(epoch): 882s + [Epoch 87][30/66] loss=0.751096 avg=0.842010 VRAM=39.5GiB | 29.2% done | ETA(epoch): 690s + [Epoch 87][40/66] loss=0.833609 avg=0.839990 VRAM=39.2GiB | 29.2% done | ETA(epoch): 499s + [Epoch 87][50/66] loss=0.885943 avg=0.851523 VRAM=39.5GiB | 29.3% done | ETA(epoch): 307s + [Epoch 87][60/66] loss=0.808521 avg=0.857254 VRAM=39.2GiB | 29.3% done | ETA(epoch): 115s + Train loss: 0.859975 (1265.6s) ETA: 4927min + Val loss: 0.886512 [t_0.0-0.2=1.0695 t_0.2-0.4=0.9786 t_0.4-0.6=0.8201 t_0.6-0.8=0.7268 t_0.8-1.0=0.6712] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 88/299 (29% done) --- + [Epoch 88][10/66] loss=0.783720 avg=0.894999 VRAM=39.5GiB | 29.4% done | ETA(epoch): 1073s + [Epoch 88][20/66] loss=0.778999 avg=0.872315 VRAM=39.2GiB | 29.4% done | ETA(epoch): 882s + [Epoch 88][30/66] loss=0.972293 avg=0.868268 VRAM=39.5GiB | 29.5% done | ETA(epoch): 690s + [Epoch 88][40/66] loss=0.826801 avg=0.877645 VRAM=39.2GiB | 29.5% done | ETA(epoch): 499s + [Epoch 88][50/66] loss=0.699838 avg=0.865924 VRAM=39.5GiB | 29.6% done | ETA(epoch): 307s + [Epoch 88][60/66] loss=0.857560 avg=0.870570 VRAM=39.2GiB | 29.6% done | ETA(epoch): 115s + Train loss: 0.864285 (1265.7s) ETA: 4903min + Val loss: 0.902534 [t_0.0-0.2=1.0916 t_0.2-0.4=1.0053 t_0.4-0.6=0.8310 t_0.6-0.8=0.7170 t_0.8-1.0=0.6506] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 89/299 (30% done) --- + [Epoch 89][10/66] loss=0.895708 avg=0.867287 VRAM=39.5GiB | 29.7% done | ETA(epoch): 1074s + [Epoch 89][20/66] loss=0.892822 avg=0.873145 VRAM=39.2GiB | 29.8% done | ETA(epoch): 882s + [Epoch 89][30/66] loss=0.767144 avg=0.880758 VRAM=39.5GiB | 29.8% done | ETA(epoch): 693s + [Epoch 89][40/66] loss=0.882147 avg=0.878878 VRAM=39.2GiB | 29.9% done | ETA(epoch): 500s + [Epoch 89][50/66] loss=0.905036 avg=0.875206 VRAM=39.5GiB | 29.9% done | ETA(epoch): 308s + [Epoch 89][60/66] loss=0.962784 avg=0.876220 VRAM=39.2GiB | 30.0% done | ETA(epoch): 115s + Train loss: 0.870386 (1268.0s) ETA: 4878min + Val loss: 0.861400 [t_0.0-0.2=1.0702 t_0.2-0.4=0.9973 t_0.4-0.6=0.8518 t_0.6-0.8=0.7091 t_0.8-1.0=0.6727] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 90/299 (30% done) --- + [MilestoneVis] train_0 step 5940 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 5940 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 5940 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 5940 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 5940 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 5940 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 5940 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 5940 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 5940 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 5940 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + [Epoch 90][10/66] loss=0.939864 avg=0.880355 VRAM=39.5GiB | 30.1% done | ETA(epoch): 6797s + [Epoch 90][20/66] loss=0.818393 avg=0.859749 VRAM=39.2GiB | 30.1% done | ETA(epoch): 3234s + [Epoch 90][30/66] loss=0.951590 avg=0.862303 VRAM=39.5GiB | 30.2% done | ETA(epoch): 1917s + [Epoch 90][40/66] loss=0.865056 avg=0.864777 VRAM=39.2GiB | 30.2% done | ETA(epoch): 1163s + [Epoch 90][50/66] loss=0.725650 avg=0.867648 VRAM=39.5GiB | 30.3% done | ETA(epoch): 634s + [Epoch 90][60/66] loss=0.916910 avg=0.863861 VRAM=39.2GiB | 30.3% done | ETA(epoch): 217s + Train loss: 0.857856 (2288.3s) ETA: 4893min + Val loss: 0.841555 [t_0.0-0.2=1.0654 t_0.2-0.4=1.0051 t_0.4-0.6=0.8700 t_0.6-0.8=0.7090 t_0.8-1.0=0.6585] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 91/299 (30% done) --- + [Epoch 91][10/66] loss=1.009181 avg=0.860606 VRAM=39.5GiB | 30.4% done | ETA(epoch): 1073s + [Epoch 91][20/66] loss=0.875842 avg=0.841112 VRAM=39.2GiB | 30.4% done | ETA(epoch): 882s + [Epoch 91][30/66] loss=0.801679 avg=0.845327 VRAM=39.5GiB | 30.5% done | ETA(epoch): 690s + [Epoch 91][40/66] loss=0.990355 avg=0.846445 VRAM=39.2GiB | 30.5% done | ETA(epoch): 499s + [Epoch 91][50/66] loss=0.964423 avg=0.850867 VRAM=39.5GiB | 30.6% done | ETA(epoch): 307s + [Epoch 91][60/66] loss=0.714415 avg=0.848621 VRAM=39.2GiB | 30.6% done | ETA(epoch): 115s + Train loss: 0.844004 (1266.3s) ETA: 4868min + Val loss: 0.833042 [t_0.0-0.2=1.0739 t_0.2-0.4=1.0115 t_0.4-0.6=0.8511 t_0.6-0.8=0.7340 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 92/299 (31% done) --- + [Epoch 92][10/66] loss=0.948652 avg=0.871643 VRAM=39.5GiB | 30.7% done | ETA(epoch): 1117s + [Epoch 92][20/66] loss=0.941859 avg=0.859658 VRAM=39.2GiB | 30.8% done | ETA(epoch): 900s + [Epoch 92][30/66] loss=0.813734 avg=0.857643 VRAM=39.5GiB | 30.8% done | ETA(epoch): 700s + [Epoch 92][40/66] loss=0.771417 avg=0.857574 VRAM=39.2GiB | 30.9% done | ETA(epoch): 504s + [Epoch 92][50/66] loss=0.778134 avg=0.861564 VRAM=39.5GiB | 30.9% done | ETA(epoch): 309s + [Epoch 92][60/66] loss=0.762875 avg=0.855799 VRAM=39.2GiB | 31.0% done | ETA(epoch): 116s + Train loss: 0.854341 (1273.3s) ETA: 4843min + Val loss: 0.864522 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9823 t_0.4-0.6=0.8779 t_0.6-0.8=0.7090 t_0.8-1.0=0.6807] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 93/299 (31% done) --- + [Epoch 93][10/66] loss=0.905192 avg=0.879878 VRAM=39.5GiB | 31.1% done | ETA(epoch): 1072s + [Epoch 93][20/66] loss=0.730288 avg=0.867831 VRAM=39.2GiB | 31.1% done | ETA(epoch): 881s + [Epoch 93][30/66] loss=0.766430 avg=0.865351 VRAM=39.5GiB | 31.2% done | ETA(epoch): 690s + [Epoch 93][40/66] loss=0.844397 avg=0.861013 VRAM=39.2GiB | 31.2% done | ETA(epoch): 498s + [Epoch 93][50/66] loss=0.926079 avg=0.857829 VRAM=39.5GiB | 31.3% done | ETA(epoch): 307s + [Epoch 93][60/66] loss=0.808120 avg=0.860310 VRAM=39.2GiB | 31.3% done | ETA(epoch): 115s + Train loss: 0.859449 (1267.9s) ETA: 4818min + Val loss: 0.882666 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9846 t_0.4-0.6=0.8102 t_0.6-0.8=0.7324 t_0.8-1.0=0.6590] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 94/299 (31% done) --- + [Epoch 94][10/66] loss=0.802929 avg=0.842750 VRAM=39.5GiB | 31.4% done | ETA(epoch): 1076s + [Epoch 94][20/66] loss=0.790522 avg=0.850218 VRAM=39.2GiB | 31.4% done | ETA(epoch): 883s + [Epoch 94][30/66] loss=0.899873 avg=0.856652 VRAM=39.5GiB | 31.5% done | ETA(epoch): 691s + [Epoch 94][40/66] loss=0.757143 avg=0.856096 VRAM=39.2GiB | 31.5% done | ETA(epoch): 499s + [Epoch 94][50/66] loss=0.947456 avg=0.856232 VRAM=39.5GiB | 31.6% done | ETA(epoch): 307s + [Epoch 94][60/66] loss=0.774794 avg=0.854675 VRAM=39.2GiB | 31.6% done | ETA(epoch): 115s + Train loss: 0.856062 (1269.7s) ETA: 4793min + Val loss: 0.873701 [t_0.0-0.2=1.0730 t_0.2-0.4=1.0123 t_0.4-0.6=0.8611 t_0.6-0.8=0.7097 t_0.8-1.0=0.6658] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 95/299 (32% done) --- + [Epoch 95][10/66] loss=0.930465 avg=0.856403 VRAM=39.5GiB | 31.7% done | ETA(epoch): 1072s + [Epoch 95][20/66] loss=0.915251 avg=0.868286 VRAM=39.2GiB | 31.8% done | ETA(epoch): 881s + [Epoch 95][30/66] loss=0.804647 avg=0.867168 VRAM=39.5GiB | 31.8% done | ETA(epoch): 690s + [Epoch 95][40/66] loss=0.807246 avg=0.866732 VRAM=39.2GiB | 31.9% done | ETA(epoch): 499s + [Epoch 95][50/66] loss=0.873312 avg=0.866225 VRAM=39.5GiB | 31.9% done | ETA(epoch): 307s + [Epoch 95][60/66] loss=0.924774 avg=0.861857 VRAM=39.2GiB | 32.0% done | ETA(epoch): 115s + Train loss: 0.859183 (1267.8s) ETA: 4768min + Val loss: 0.869197 [t_0.0-0.2=1.0682 t_0.2-0.4=0.9892 t_0.4-0.6=0.8384 t_0.6-0.8=0.7365 t_0.8-1.0=0.6847] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 96/299 (32% done) --- + [Epoch 96][10/66] loss=0.965122 avg=0.899552 VRAM=39.5GiB | 32.1% done | ETA(epoch): 1074s + [Epoch 96][20/66] loss=0.779539 avg=0.859376 VRAM=39.2GiB | 32.1% done | ETA(epoch): 882s + [Epoch 96][30/66] loss=0.847741 avg=0.850250 VRAM=39.5GiB | 32.2% done | ETA(epoch): 690s + [Epoch 96][40/66] loss=0.908521 avg=0.844027 VRAM=39.2GiB | 32.2% done | ETA(epoch): 500s + [Epoch 96][50/66] loss=0.774372 avg=0.846522 VRAM=39.5GiB | 32.3% done | ETA(epoch): 307s + [Epoch 96][60/66] loss=0.856315 avg=0.853787 VRAM=39.2GiB | 32.3% done | ETA(epoch): 115s + Train loss: 0.858425 (1267.7s) ETA: 4744min + Val loss: 0.860458 [t_0.0-0.2=1.0691 t_0.2-0.4=0.9750 t_0.4-0.6=0.8276 t_0.6-0.8=0.7068 t_0.8-1.0=0.6674] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 97/299 (32% done) --- + [Epoch 97][10/66] loss=0.878271 avg=0.873281 VRAM=39.5GiB | 32.4% done | ETA(epoch): 1077s + [Epoch 97][20/66] loss=0.881319 avg=0.864889 VRAM=39.2GiB | 32.4% done | ETA(epoch): 884s + [Epoch 97][30/66] loss=0.834731 avg=0.856299 VRAM=39.5GiB | 32.5% done | ETA(epoch): 691s + [Epoch 97][40/66] loss=0.863333 avg=0.849647 VRAM=39.2GiB | 32.5% done | ETA(epoch): 499s + [Epoch 97][50/66] loss=0.871367 avg=0.850867 VRAM=39.5GiB | 32.6% done | ETA(epoch): 307s + [Epoch 97][60/66] loss=0.882621 avg=0.854857 VRAM=39.2GiB | 32.6% done | ETA(epoch): 115s + Train loss: 0.855113 (1266.2s) ETA: 4719min + Val loss: 0.858825 [t_0.0-0.2=1.0900 t_0.2-0.4=1.0195 t_0.4-0.6=0.8352 t_0.6-0.8=0.7043 t_0.8-1.0=0.6557] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 98/299 (33% done) --- + [Epoch 98][10/66] loss=0.950565 avg=0.831745 VRAM=39.5GiB | 32.7% done | ETA(epoch): 1073s + [Epoch 98][20/66] loss=0.859969 avg=0.846304 VRAM=39.2GiB | 32.8% done | ETA(epoch): 881s + [Epoch 98][30/66] loss=0.851337 avg=0.834869 VRAM=39.5GiB | 32.8% done | ETA(epoch): 690s + [Epoch 98][40/66] loss=0.909922 avg=0.843018 VRAM=39.2GiB | 32.9% done | ETA(epoch): 498s + [Epoch 98][50/66] loss=0.895026 avg=0.837843 VRAM=39.5GiB | 32.9% done | ETA(epoch): 307s + [Epoch 98][60/66] loss=0.804161 avg=0.837348 VRAM=39.2GiB | 33.0% done | ETA(epoch): 115s + Train loss: 0.839434 (1265.5s) ETA: 4694min + Val loss: 0.848834 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9813 t_0.4-0.6=0.8057 t_0.6-0.8=0.7260 t_0.8-1.0=0.6582] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 99/299 (33% done) --- + [Epoch 99][10/66] loss=0.758191 avg=0.842471 VRAM=39.5GiB | 33.1% done | ETA(epoch): 1072s + [Epoch 99][20/66] loss=0.762522 avg=0.835961 VRAM=39.2GiB | 33.1% done | ETA(epoch): 882s + [Epoch 99][30/66] loss=0.960355 avg=0.839486 VRAM=39.5GiB | 33.2% done | ETA(epoch): 691s + [Epoch 99][40/66] loss=0.936442 avg=0.851283 VRAM=39.2GiB | 33.2% done | ETA(epoch): 499s + [Epoch 99][50/66] loss=0.825390 avg=0.849286 VRAM=39.5GiB | 33.3% done | ETA(epoch): 307s + [Epoch 99][60/66] loss=0.801657 avg=0.854280 VRAM=39.2GiB | 33.3% done | ETA(epoch): 115s + Train loss: 0.859345 (1265.5s) ETA: 4669min + Val loss: 0.876873 [t_0.0-0.2=1.0701 t_0.2-0.4=0.9995 t_0.4-0.6=0.8444 t_0.6-0.8=0.7152 t_0.8-1.0=0.6544] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 100/299 (33% done) --- + [Epoch 100][10/66] loss=1.002892 avg=0.852738 VRAM=39.5GiB | 33.4% done | ETA(epoch): 1104s + [Epoch 100][20/66] loss=0.892363 avg=0.867470 VRAM=39.2GiB | 33.4% done | ETA(epoch): 895s + [Epoch 100][30/66] loss=0.943002 avg=0.860755 VRAM=39.5GiB | 33.5% done | ETA(epoch): 697s + [Epoch 100][40/66] loss=0.801685 avg=0.865910 VRAM=39.2GiB | 33.5% done | ETA(epoch): 502s + [Epoch 100][50/66] loss=0.700122 avg=0.862352 VRAM=39.5GiB | 33.6% done | ETA(epoch): 309s + [Epoch 100][60/66] loss=0.852914 avg=0.863239 VRAM=39.2GiB | 33.6% done | ETA(epoch): 116s + Train loss: 0.857245 (1271.2s) ETA: 4645min + Val loss: 0.852588 [t_0.0-0.2=1.0590 t_0.2-0.4=1.0169 t_0.4-0.6=0.8436 t_0.6-0.8=0.7080 t_0.8-1.0=0.6636] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 101/299 (34% done) --- + [Epoch 101][10/66] loss=0.800631 avg=0.857427 VRAM=39.5GiB | 33.7% done | ETA(epoch): 1074s + [Epoch 101][20/66] loss=0.748188 avg=0.856632 VRAM=39.2GiB | 33.8% done | ETA(epoch): 882s + [Epoch 101][30/66] loss=0.705169 avg=0.859626 VRAM=39.5GiB | 33.8% done | ETA(epoch): 690s + [Epoch 101][40/66] loss=0.854057 avg=0.859128 VRAM=39.2GiB | 33.9% done | ETA(epoch): 498s + [Epoch 101][50/66] loss=0.871778 avg=0.865277 VRAM=39.5GiB | 33.9% done | ETA(epoch): 307s + [Epoch 101][60/66] loss=0.918074 avg=0.867302 VRAM=39.2GiB | 34.0% done | ETA(epoch): 115s + Train loss: 0.863698 (1269.1s) ETA: 4620min + Val loss: 0.839153 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0279 t_0.4-0.6=0.8369 t_0.6-0.8=0.7202 t_0.8-1.0=0.6612] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 102/299 (34% done) --- + [Epoch 102][10/66] loss=0.874179 avg=0.855574 VRAM=39.5GiB | 34.1% done | ETA(epoch): 1073s + [Epoch 102][20/66] loss=1.062593 avg=0.869510 VRAM=39.2GiB | 34.1% done | ETA(epoch): 881s + [Epoch 102][30/66] loss=0.664581 avg=0.858662 VRAM=39.5GiB | 34.2% done | ETA(epoch): 690s + [Epoch 102][40/66] loss=0.871148 avg=0.864532 VRAM=39.2GiB | 34.2% done | ETA(epoch): 498s + [Epoch 102][50/66] loss=0.924993 avg=0.866757 VRAM=39.5GiB | 34.3% done | ETA(epoch): 307s + [Epoch 102][60/66] loss=0.718019 avg=0.866654 VRAM=39.2GiB | 34.3% done | ETA(epoch): 115s + Train loss: 0.866018 (1267.0s) ETA: 4596min + Val loss: 0.856230 [t_0.0-0.2=1.0638 t_0.2-0.4=1.0262 t_0.4-0.6=0.8513 t_0.6-0.8=0.7332 t_0.8-1.0=0.6909] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 103/299 (34% done) --- + [Epoch 103][10/66] loss=0.815737 avg=0.853616 VRAM=39.5GiB | 34.4% done | ETA(epoch): 1074s + [Epoch 103][20/66] loss=0.756133 avg=0.862822 VRAM=39.2GiB | 34.4% done | ETA(epoch): 883s + [Epoch 103][30/66] loss=0.807715 avg=0.874747 VRAM=39.5GiB | 34.5% done | ETA(epoch): 691s + [Epoch 103][40/66] loss=0.885679 avg=0.866206 VRAM=39.2GiB | 34.5% done | ETA(epoch): 499s + [Epoch 103][50/66] loss=0.952237 avg=0.865012 VRAM=39.5GiB | 34.6% done | ETA(epoch): 308s + [Epoch 103][60/66] loss=0.785868 avg=0.861695 VRAM=39.2GiB | 34.6% done | ETA(epoch): 115s + Train loss: 0.858753 (1267.7s) ETA: 4571min + Val loss: 0.834117 [t_0.0-0.2=1.0665 t_0.2-0.4=0.9692 t_0.4-0.6=0.8274 t_0.6-0.8=0.7422 t_0.8-1.0=0.6687] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 18.8/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 104/299 (35% done) --- + [Epoch 104][10/66] loss=0.866244 avg=0.856426 VRAM=39.5GiB | 34.7% done | ETA(epoch): 1073s + [Epoch 104][20/66] loss=0.780788 avg=0.844370 VRAM=39.2GiB | 34.8% done | ETA(epoch): 882s + [Epoch 104][30/66] loss=0.940126 avg=0.849357 VRAM=39.5GiB | 34.8% done | ETA(epoch): 689s + [Epoch 104][40/66] loss=0.891194 avg=0.854142 VRAM=39.2GiB | 34.9% done | ETA(epoch): 498s + [Epoch 104][50/66] loss=0.810441 avg=0.855558 VRAM=39.5GiB | 34.9% done | ETA(epoch): 307s + [Epoch 104][60/66] loss=0.926783 avg=0.864610 VRAM=39.2GiB | 35.0% done | ETA(epoch): 115s + Train loss: 0.868550 (1265.0s) ETA: 4547min + Val loss: 0.861740 [t_0.0-0.2=1.0541 t_0.2-0.4=0.9999 t_0.4-0.6=0.8791 t_0.6-0.8=0.7182 t_0.8-1.0=0.6838] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 105/299 (35% done) --- + [Epoch 105][10/66] loss=0.780085 avg=0.821315 VRAM=39.5GiB | 35.1% done | ETA(epoch): 1074s + [Epoch 105][20/66] loss=0.953508 avg=0.846677 VRAM=39.2GiB | 35.1% done | ETA(epoch): 882s + [Epoch 105][30/66] loss=0.792367 avg=0.854223 VRAM=39.5GiB | 35.2% done | ETA(epoch): 690s + [Epoch 105][40/66] loss=0.762063 avg=0.851282 VRAM=39.2GiB | 35.2% done | ETA(epoch): 498s + [Epoch 105][50/66] loss=0.853601 avg=0.856236 VRAM=39.5GiB | 35.3% done | ETA(epoch): 307s + [Epoch 105][60/66] loss=0.799999 avg=0.855813 VRAM=39.2GiB | 35.3% done | ETA(epoch): 115s + Train loss: 0.855208 (1265.2s) ETA: 4522min + Val loss: 0.858651 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9928 t_0.4-0.6=0.8632 t_0.6-0.8=0.7022 t_0.8-1.0=0.6765] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 106/299 (35% done) --- + [Epoch 106][10/66] loss=0.897841 avg=0.839533 VRAM=39.5GiB | 35.4% done | ETA(epoch): 1072s + [Epoch 106][20/66] loss=0.860660 avg=0.850683 VRAM=39.2GiB | 35.4% done | ETA(epoch): 898s + [Epoch 106][30/66] loss=0.907982 avg=0.848542 VRAM=39.5GiB | 35.5% done | ETA(epoch): 699s + [Epoch 106][40/66] loss=0.984564 avg=0.858042 VRAM=39.2GiB | 35.5% done | ETA(epoch): 503s + [Epoch 106][50/66] loss=0.952395 avg=0.861974 VRAM=39.5GiB | 35.6% done | ETA(epoch): 309s + [Epoch 106][60/66] loss=0.882165 avg=0.859065 VRAM=39.2GiB | 35.6% done | ETA(epoch): 116s + Train loss: 0.858671 (1273.0s) ETA: 4498min + Val loss: 0.890271 [t_0.0-0.2=1.0747 t_0.2-0.4=1.0013 t_0.4-0.6=0.8800 t_0.6-0.8=0.7155 t_0.8-1.0=0.6647] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 107/299 (36% done) --- + [Epoch 107][10/66] loss=0.764714 avg=0.838325 VRAM=39.5GiB | 35.7% done | ETA(epoch): 1072s + [Epoch 107][20/66] loss=0.905366 avg=0.846200 VRAM=39.2GiB | 35.8% done | ETA(epoch): 893s + [Epoch 107][30/66] loss=0.804693 avg=0.846873 VRAM=39.5GiB | 35.8% done | ETA(epoch): 697s + [Epoch 107][40/66] loss=0.923037 avg=0.848037 VRAM=39.2GiB | 35.9% done | ETA(epoch): 502s + [Epoch 107][50/66] loss=0.890434 avg=0.850947 VRAM=39.5GiB | 35.9% done | ETA(epoch): 308s + [Epoch 107][60/66] loss=0.935280 avg=0.849804 VRAM=39.2GiB | 36.0% done | ETA(epoch): 116s + Train loss: 0.847645 (1271.1s) ETA: 4474min + Val loss: 0.889654 [t_0.0-0.2=1.0872 t_0.2-0.4=0.9785 t_0.4-0.6=0.8516 t_0.6-0.8=0.7169 t_0.8-1.0=0.6626] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 108/299 (36% done) --- + [Epoch 108][10/66] loss=0.835943 avg=0.846771 VRAM=39.5GiB | 36.1% done | ETA(epoch): 1086s + [Epoch 108][20/66] loss=0.882195 avg=0.847398 VRAM=39.2GiB | 36.1% done | ETA(epoch): 886s + [Epoch 108][30/66] loss=0.907683 avg=0.846276 VRAM=39.5GiB | 36.2% done | ETA(epoch): 692s + [Epoch 108][40/66] loss=0.777968 avg=0.844287 VRAM=39.2GiB | 36.2% done | ETA(epoch): 500s + [Epoch 108][50/66] loss=0.738506 avg=0.852637 VRAM=39.5GiB | 36.3% done | ETA(epoch): 307s + [Epoch 108][60/66] loss=0.987133 avg=0.857204 VRAM=39.2GiB | 36.3% done | ETA(epoch): 115s + Train loss: 0.855719 (1268.9s) ETA: 4449min + Val loss: 0.889138 [t_0.0-0.2=1.0678 t_0.2-0.4=1.0044 t_0.4-0.6=0.8395 t_0.6-0.8=0.7188 t_0.8-1.0=0.6722] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 109/299 (36% done) --- + [Epoch 109][10/66] loss=0.801159 avg=0.870525 VRAM=39.5GiB | 36.4% done | ETA(epoch): 1072s + [Epoch 109][20/66] loss=0.881911 avg=0.868935 VRAM=39.2GiB | 36.4% done | ETA(epoch): 882s + [Epoch 109][30/66] loss=0.784714 avg=0.851329 VRAM=39.5GiB | 36.5% done | ETA(epoch): 690s + [Epoch 109][40/66] loss=0.909995 avg=0.861648 VRAM=39.2GiB | 36.5% done | ETA(epoch): 499s + [Epoch 109][50/66] loss=0.774512 avg=0.856807 VRAM=39.5GiB | 36.6% done | ETA(epoch): 307s + [Epoch 109][60/66] loss=0.982868 avg=0.861589 VRAM=39.2GiB | 36.6% done | ETA(epoch): 115s + Train loss: 0.859583 (1266.2s) ETA: 4425min + Val loss: 0.850844 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9788 t_0.4-0.6=0.8100 t_0.6-0.8=0.7244 t_0.8-1.0=0.6621] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 110/299 (37% done) --- + [Epoch 110][10/66] loss=0.907210 avg=0.861922 VRAM=39.5GiB | 36.7% done | ETA(epoch): 1073s + [Epoch 110][20/66] loss=0.770006 avg=0.871828 VRAM=39.2GiB | 36.8% done | ETA(epoch): 881s + [Epoch 110][30/66] loss=0.962745 avg=0.880960 VRAM=39.5GiB | 36.8% done | ETA(epoch): 690s + [Epoch 110][40/66] loss=0.809191 avg=0.874267 VRAM=39.2GiB | 36.9% done | ETA(epoch): 498s + [Epoch 110][50/66] loss=0.846399 avg=0.866992 VRAM=39.5GiB | 36.9% done | ETA(epoch): 307s + [Epoch 110][60/66] loss=0.768030 avg=0.860813 VRAM=39.2GiB | 37.0% done | ETA(epoch): 115s + Train loss: 0.865479 (1265.6s) ETA: 4401min + Val loss: 0.847008 [t_0.0-0.2=1.0848 t_0.2-0.4=0.9798 t_0.4-0.6=0.8656 t_0.6-0.8=0.7241 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 111/299 (37% done) --- + [Epoch 111][10/66] loss=0.819562 avg=0.903463 VRAM=39.5GiB | 37.1% done | ETA(epoch): 1075s + [Epoch 111][20/66] loss=0.810027 avg=0.884366 VRAM=39.2GiB | 37.1% done | ETA(epoch): 883s + [Epoch 111][30/66] loss=0.864114 avg=0.875643 VRAM=39.5GiB | 37.2% done | ETA(epoch): 691s + [Epoch 111][40/66] loss=0.851861 avg=0.865893 VRAM=39.2GiB | 37.2% done | ETA(epoch): 499s + [Epoch 111][50/66] loss=0.906538 avg=0.865854 VRAM=39.5GiB | 37.3% done | ETA(epoch): 309s + [Epoch 111][60/66] loss=0.898925 avg=0.861471 VRAM=39.2GiB | 37.3% done | ETA(epoch): 116s + Train loss: 0.861242 (1273.1s) ETA: 4377min + Val loss: 0.869159 [t_0.0-0.2=1.0800 t_0.2-0.4=0.9970 t_0.4-0.6=0.8367 t_0.6-0.8=0.7275 t_0.8-1.0=0.6596] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 112/299 (37% done) --- + [Epoch 112][10/66] loss=0.889967 avg=0.849255 VRAM=39.5GiB | 37.4% done | ETA(epoch): 1076s + [Epoch 112][20/66] loss=0.906068 avg=0.863182 VRAM=39.2GiB | 37.4% done | ETA(epoch): 883s + [Epoch 112][30/66] loss=0.942819 avg=0.878556 VRAM=39.5GiB | 37.5% done | ETA(epoch): 691s + [Epoch 112][40/66] loss=0.802482 avg=0.871008 VRAM=39.2GiB | 37.5% done | ETA(epoch): 503s + [Epoch 112][50/66] loss=0.753377 avg=0.866750 VRAM=39.5GiB | 37.6% done | ETA(epoch): 309s + [Epoch 112][60/66] loss=0.890597 avg=0.871455 VRAM=39.2GiB | 37.6% done | ETA(epoch): 116s + Train loss: 0.868832 (1273.4s) ETA: 4353min + Val loss: 0.840804 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0011 t_0.4-0.6=0.8450 t_0.6-0.8=0.7315 t_0.8-1.0=0.6634] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 113/299 (38% done) --- + [Epoch 113][10/66] loss=0.707435 avg=0.825875 VRAM=39.5GiB | 37.7% done | ETA(epoch): 1073s + [Epoch 113][20/66] loss=0.891619 avg=0.822852 VRAM=39.2GiB | 37.8% done | ETA(epoch): 882s + [Epoch 113][30/66] loss=0.857297 avg=0.836824 VRAM=39.5GiB | 37.8% done | ETA(epoch): 693s + [Epoch 113][40/66] loss=0.877012 avg=0.840894 VRAM=39.2GiB | 37.9% done | ETA(epoch): 500s + [Epoch 113][50/66] loss=0.953307 avg=0.847818 VRAM=39.5GiB | 37.9% done | ETA(epoch): 308s + [Epoch 113][60/66] loss=0.781725 avg=0.846002 VRAM=39.2GiB | 38.0% done | ETA(epoch): 115s + Train loss: 0.844397 (1268.5s) ETA: 4328min + Val loss: 0.835915 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9944 t_0.4-0.6=0.8214 t_0.6-0.8=0.7208 t_0.8-1.0=0.6823] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 114/299 (38% done) --- + [Epoch 114][10/66] loss=0.823291 avg=0.857117 VRAM=39.5GiB | 38.1% done | ETA(epoch): 1075s + [Epoch 114][20/66] loss=0.883491 avg=0.862306 VRAM=39.2GiB | 38.1% done | ETA(epoch): 892s + [Epoch 114][30/66] loss=0.950068 avg=0.881720 VRAM=39.5GiB | 38.2% done | ETA(epoch): 695s + [Epoch 114][40/66] loss=0.878306 avg=0.877905 VRAM=39.2GiB | 38.2% done | ETA(epoch): 502s + [Epoch 114][50/66] loss=0.973150 avg=0.879893 VRAM=39.5GiB | 38.3% done | ETA(epoch): 308s + [Epoch 114][60/66] loss=0.838026 avg=0.872847 VRAM=39.2GiB | 38.3% done | ETA(epoch): 116s + Train loss: 0.872464 (1270.6s) ETA: 4304min + Val loss: 0.883783 [t_0.0-0.2=1.0568 t_0.2-0.4=1.0157 t_0.4-0.6=0.8459 t_0.6-0.8=0.7185 t_0.8-1.0=0.6599] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 115/299 (38% done) --- + [Epoch 115][10/66] loss=0.929282 avg=0.872970 VRAM=39.5GiB | 38.4% done | ETA(epoch): 1075s + [Epoch 115][20/66] loss=0.852271 avg=0.852560 VRAM=39.2GiB | 38.4% done | ETA(epoch): 884s + [Epoch 115][30/66] loss=1.007780 avg=0.869015 VRAM=39.5GiB | 38.5% done | ETA(epoch): 693s + [Epoch 115][40/66] loss=0.886108 avg=0.864687 VRAM=39.2GiB | 38.5% done | ETA(epoch): 500s + [Epoch 115][50/66] loss=0.810155 avg=0.865141 VRAM=39.5GiB | 38.6% done | ETA(epoch): 308s + [Epoch 115][60/66] loss=0.947926 avg=0.868266 VRAM=39.2GiB | 38.6% done | ETA(epoch): 115s + Train loss: 0.868450 (1268.4s) ETA: 4280min + Val loss: 0.853173 [t_0.0-0.2=1.0705 t_0.2-0.4=1.0122 t_0.4-0.6=0.8707 t_0.6-0.8=0.7126 t_0.8-1.0=0.6599] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 116/299 (39% done) --- + [Epoch 116][10/66] loss=0.794785 avg=0.848466 VRAM=39.5GiB | 38.7% done | ETA(epoch): 1076s + [Epoch 116][20/66] loss=0.969175 avg=0.874069 VRAM=39.2GiB | 38.8% done | ETA(epoch): 883s + [Epoch 116][30/66] loss=1.017481 avg=0.873470 VRAM=39.5GiB | 38.8% done | ETA(epoch): 691s + [Epoch 116][40/66] loss=0.798530 avg=0.867568 VRAM=39.2GiB | 38.9% done | ETA(epoch): 499s + [Epoch 116][50/66] loss=1.031439 avg=0.864364 VRAM=39.5GiB | 38.9% done | ETA(epoch): 307s + [Epoch 116][60/66] loss=0.793918 avg=0.857321 VRAM=39.2GiB | 39.0% done | ETA(epoch): 115s + Train loss: 0.859769 (1271.6s) ETA: 4256min + Val loss: 0.849501 [t_0.0-0.2=1.0599 t_0.2-0.4=1.0128 t_0.4-0.6=0.8207 t_0.6-0.8=0.7085 t_0.8-1.0=0.6584] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 18.2/188.5 GiB (9.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 117/299 (39% done) --- + [Epoch 117][10/66] loss=0.982418 avg=0.916139 VRAM=39.5GiB | 39.1% done | ETA(epoch): 1076s + [Epoch 117][20/66] loss=0.902967 avg=0.895462 VRAM=39.2GiB | 39.1% done | ETA(epoch): 885s + [Epoch 117][30/66] loss=0.783290 avg=0.873218 VRAM=39.5GiB | 39.2% done | ETA(epoch): 692s + [Epoch 117][40/66] loss=0.986251 avg=0.874512 VRAM=39.2GiB | 39.2% done | ETA(epoch): 500s + [Epoch 117][50/66] loss=0.852540 avg=0.870489 VRAM=39.5GiB | 39.3% done | ETA(epoch): 308s + [Epoch 117][60/66] loss=0.813771 avg=0.864228 VRAM=39.2GiB | 39.3% done | ETA(epoch): 116s + Train loss: 0.862132 (1272.2s) ETA: 4232min + Val loss: 0.830254 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9879 t_0.4-0.6=0.8335 t_0.6-0.8=0.6976 t_0.8-1.0=0.6732] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 118/299 (39% done) --- + [Epoch 118][10/66] loss=0.748021 avg=0.834158 VRAM=39.5GiB | 39.4% done | ETA(epoch): 1078s + [Epoch 118][20/66] loss=0.789034 avg=0.842934 VRAM=39.2GiB | 39.4% done | ETA(epoch): 885s + [Epoch 118][30/66] loss=0.864763 avg=0.841479 VRAM=39.5GiB | 39.5% done | ETA(epoch): 693s + [Epoch 118][40/66] loss=0.901498 avg=0.852137 VRAM=39.2GiB | 39.5% done | ETA(epoch): 501s + [Epoch 118][50/66] loss=0.925769 avg=0.857436 VRAM=39.5GiB | 39.6% done | ETA(epoch): 308s + [Epoch 118][60/66] loss=0.875391 avg=0.859723 VRAM=39.2GiB | 39.6% done | ETA(epoch): 116s + Train loss: 0.864906 (1270.9s) ETA: 4208min + Val loss: 0.853338 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9914 t_0.4-0.6=0.8330 t_0.6-0.8=0.7084 t_0.8-1.0=0.6696] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 18.1/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 119/299 (40% done) --- + [Epoch 119][10/66] loss=0.891266 avg=0.836754 VRAM=39.5GiB | 39.7% done | ETA(epoch): 1077s + [Epoch 119][20/66] loss=0.864690 avg=0.847511 VRAM=39.2GiB | 39.8% done | ETA(epoch): 885s + [Epoch 119][30/66] loss=0.755306 avg=0.856421 VRAM=39.5GiB | 39.8% done | ETA(epoch): 693s + [Epoch 119][40/66] loss=0.837836 avg=0.861953 VRAM=39.2GiB | 39.9% done | ETA(epoch): 501s + [Epoch 119][50/66] loss=0.841711 avg=0.868837 VRAM=39.5GiB | 39.9% done | ETA(epoch): 308s + [Epoch 119][60/66] loss=0.834105 avg=0.862978 VRAM=39.2GiB | 40.0% done | ETA(epoch): 116s + Train loss: 0.861494 (1271.3s) ETA: 4184min + Val loss: 0.851840 [t_0.0-0.2=1.0661 t_0.2-0.4=0.9917 t_0.4-0.6=0.8285 t_0.6-0.8=0.7193 t_0.8-1.0=0.6693] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 18.2/188.5 GiB (9.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 120/299 (40% done) --- + [MilestoneVis] train_0 step 7920 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 7920 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 7920 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 7920 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 7920 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 7920 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 7920 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 7920 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 7920 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 7920 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + [Epoch 120][10/66] loss=0.797426 avg=0.798301 VRAM=39.5GiB | 40.1% done | ETA(epoch): 6835s + [Epoch 120][20/66] loss=0.722352 avg=0.824432 VRAM=39.2GiB | 40.1% done | ETA(epoch): 3250s + [Epoch 120][30/66] loss=0.922483 avg=0.855761 VRAM=39.5GiB | 40.2% done | ETA(epoch): 1926s + [Epoch 120][40/66] loss=0.800863 avg=0.851215 VRAM=39.2GiB | 40.2% done | ETA(epoch): 1170s + [Epoch 120][50/66] loss=0.881391 avg=0.853142 VRAM=39.5GiB | 40.3% done | ETA(epoch): 637s + [Epoch 120][60/66] loss=0.822396 avg=0.862567 VRAM=39.2GiB | 40.3% done | ETA(epoch): 218s + Train loss: 0.863203 (2300.2s) ETA: 4185min + Val loss: 0.843931 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0111 t_0.4-0.6=0.8194 t_0.6-0.8=0.7183 t_0.8-1.0=0.6546] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 121/299 (40% done) --- + [Epoch 121][10/66] loss=0.875510 avg=0.847802 VRAM=39.5GiB | 40.4% done | ETA(epoch): 1079s + [Epoch 121][20/66] loss=0.872720 avg=0.847824 VRAM=39.2GiB | 40.4% done | ETA(epoch): 887s + [Epoch 121][30/66] loss=0.792746 avg=0.853100 VRAM=39.5GiB | 40.5% done | ETA(epoch): 694s + [Epoch 121][40/66] loss=0.746685 avg=0.855863 VRAM=39.2GiB | 40.5% done | ETA(epoch): 501s + [Epoch 121][50/66] loss=0.844450 avg=0.856846 VRAM=39.5GiB | 40.6% done | ETA(epoch): 309s + [Epoch 121][60/66] loss=0.874024 avg=0.853184 VRAM=39.2GiB | 40.6% done | ETA(epoch): 116s + Train loss: 0.853855 (1272.7s) ETA: 4161min + Val loss: 0.870921 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9905 t_0.4-0.6=0.8477 t_0.6-0.8=0.7331 t_0.8-1.0=0.6746] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 122/299 (41% done) --- + [Epoch 122][10/66] loss=0.757579 avg=0.828263 VRAM=39.5GiB | 40.7% done | ETA(epoch): 1077s + [Epoch 122][20/66] loss=0.768335 avg=0.835503 VRAM=39.2GiB | 40.8% done | ETA(epoch): 885s + [Epoch 122][30/66] loss=0.819126 avg=0.839854 VRAM=39.5GiB | 40.8% done | ETA(epoch): 695s + [Epoch 122][40/66] loss=0.811788 avg=0.837221 VRAM=39.2GiB | 40.9% done | ETA(epoch): 502s + [Epoch 122][50/66] loss=0.800683 avg=0.833891 VRAM=39.5GiB | 40.9% done | ETA(epoch): 309s + [Epoch 122][60/66] loss=0.754898 avg=0.832289 VRAM=39.2GiB | 41.0% done | ETA(epoch): 116s + Train loss: 0.832282 (1272.8s) ETA: 4137min + Val loss: 0.845081 [t_0.0-0.2=1.0920 t_0.2-0.4=1.0046 t_0.4-0.6=0.8610 t_0.6-0.8=0.7346 t_0.8-1.0=0.6589] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 123/299 (41% done) --- + [Epoch 123][10/66] loss=0.836413 avg=0.835205 VRAM=39.5GiB | 41.1% done | ETA(epoch): 1079s + [Epoch 123][20/66] loss=0.802762 avg=0.858452 VRAM=39.2GiB | 41.1% done | ETA(epoch): 886s + [Epoch 123][30/66] loss=0.958708 avg=0.859273 VRAM=39.5GiB | 41.2% done | ETA(epoch): 693s + [Epoch 123][40/66] loss=0.937409 avg=0.852763 VRAM=39.2GiB | 41.2% done | ETA(epoch): 501s + [Epoch 123][50/66] loss=0.806787 avg=0.852909 VRAM=39.5GiB | 41.3% done | ETA(epoch): 308s + [Epoch 123][60/66] loss=0.825110 avg=0.859497 VRAM=39.2GiB | 41.3% done | ETA(epoch): 116s + Train loss: 0.860692 (1271.7s) ETA: 4113min + Val loss: 0.875469 [t_0.0-0.2=1.0670 t_0.2-0.4=0.9740 t_0.4-0.6=0.8553 t_0.6-0.8=0.7050 t_0.8-1.0=0.6550] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 124/299 (41% done) --- + [Epoch 124][10/66] loss=0.875393 avg=0.871831 VRAM=39.5GiB | 41.4% done | ETA(epoch): 1079s + [Epoch 124][20/66] loss=0.855139 avg=0.879841 VRAM=39.2GiB | 41.4% done | ETA(epoch): 886s + [Epoch 124][30/66] loss=0.991259 avg=0.878972 VRAM=39.5GiB | 41.5% done | ETA(epoch): 694s + [Epoch 124][40/66] loss=0.827401 avg=0.875019 VRAM=39.2GiB | 41.5% done | ETA(epoch): 501s + [Epoch 124][50/66] loss=0.816369 avg=0.865828 VRAM=39.5GiB | 41.6% done | ETA(epoch): 308s + [Epoch 124][60/66] loss=0.857304 avg=0.865606 VRAM=39.2GiB | 41.6% done | ETA(epoch): 116s + Train loss: 0.857446 (1272.1s) ETA: 4089min + Val loss: 0.834501 [t_0.0-0.2=1.0525 t_0.2-0.4=0.9959 t_0.4-0.6=0.8409 t_0.6-0.8=0.7083 t_0.8-1.0=0.6605] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 125/299 (42% done) --- + [Epoch 125][10/66] loss=0.902298 avg=0.876242 VRAM=39.5GiB | 41.7% done | ETA(epoch): 1078s + [Epoch 125][20/66] loss=0.806566 avg=0.874637 VRAM=39.2GiB | 41.8% done | ETA(epoch): 886s + [Epoch 125][30/66] loss=0.943468 avg=0.865994 VRAM=39.5GiB | 41.8% done | ETA(epoch): 693s + [Epoch 125][40/66] loss=0.819693 avg=0.867056 VRAM=39.2GiB | 41.9% done | ETA(epoch): 500s + [Epoch 125][50/66] loss=0.823874 avg=0.868820 VRAM=39.5GiB | 41.9% done | ETA(epoch): 308s + [Epoch 125][60/66] loss=0.796402 avg=0.860750 VRAM=39.2GiB | 42.0% done | ETA(epoch): 116s + Train loss: 0.862567 (1271.5s) ETA: 4064min + Val loss: 0.863244 [t_0.0-0.2=1.0661 t_0.2-0.4=1.0156 t_0.4-0.6=0.8568 t_0.6-0.8=0.7031 t_0.8-1.0=0.6709] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 126/299 (42% done) --- + [Epoch 126][10/66] loss=0.899093 avg=0.900856 VRAM=39.5GiB | 42.1% done | ETA(epoch): 1078s + [Epoch 126][20/66] loss=0.926120 avg=0.879692 VRAM=39.2GiB | 42.1% done | ETA(epoch): 885s + [Epoch 126][30/66] loss=0.827100 avg=0.874300 VRAM=39.5GiB | 42.2% done | ETA(epoch): 693s + [Epoch 126][40/66] loss=0.894220 avg=0.880290 VRAM=39.2GiB | 42.2% done | ETA(epoch): 501s + [Epoch 126][50/66] loss=0.990719 avg=0.882513 VRAM=39.5GiB | 42.3% done | ETA(epoch): 308s + [Epoch 126][60/66] loss=0.820815 avg=0.878533 VRAM=39.2GiB | 42.3% done | ETA(epoch): 116s + Train loss: 0.876360 (1273.1s) ETA: 4040min + Val loss: 0.819299 [t_0.0-0.2=1.0812 t_0.2-0.4=0.9990 t_0.4-0.6=0.8311 t_0.6-0.8=0.7051 t_0.8-1.0=0.6499] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0126 (BEST) + Deleted old checkpoint: checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 127/299 (42% done) --- + [Epoch 127][10/66] loss=0.774753 avg=0.815308 VRAM=39.5GiB | 42.4% done | ETA(epoch): 1080s + [Epoch 127][20/66] loss=0.852601 avg=0.828602 VRAM=39.2GiB | 42.4% done | ETA(epoch): 887s + [Epoch 127][30/66] loss=0.969038 avg=0.837401 VRAM=39.5GiB | 42.5% done | ETA(epoch): 694s + [Epoch 127][40/66] loss=0.777914 avg=0.834070 VRAM=39.2GiB | 42.5% done | ETA(epoch): 501s + [Epoch 127][50/66] loss=0.835001 avg=0.840351 VRAM=39.5GiB | 42.6% done | ETA(epoch): 309s + [Epoch 127][60/66] loss=0.784364 avg=0.842214 VRAM=39.2GiB | 42.6% done | ETA(epoch): 116s + Train loss: 0.844190 (1272.5s) ETA: 4016min + Val loss: 0.866337 [t_0.0-0.2=1.0721 t_0.2-0.4=0.9967 t_0.4-0.6=0.8324 t_0.6-0.8=0.7026 t_0.8-1.0=0.6464] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 128/299 (43% done) --- + [Epoch 128][10/66] loss=0.762916 avg=0.840561 VRAM=39.5GiB | 42.7% done | ETA(epoch): 1078s + [Epoch 128][20/66] loss=0.727345 avg=0.853386 VRAM=39.2GiB | 42.8% done | ETA(epoch): 887s + [Epoch 128][30/66] loss=0.884182 avg=0.860859 VRAM=39.5GiB | 42.8% done | ETA(epoch): 695s + [Epoch 128][40/66] loss=0.909277 avg=0.858927 VRAM=39.2GiB | 42.9% done | ETA(epoch): 502s + [Epoch 128][50/66] loss=0.967809 avg=0.857726 VRAM=39.5GiB | 42.9% done | ETA(epoch): 309s + [Epoch 128][60/66] loss=0.859150 avg=0.851597 VRAM=39.2GiB | 43.0% done | ETA(epoch): 116s + Train loss: 0.850687 (1273.4s) ETA: 3992min + Val loss: 0.856959 [t_0.0-0.2=1.0833 t_0.2-0.4=0.9998 t_0.4-0.6=0.8236 t_0.6-0.8=0.6949 t_0.8-1.0=0.6640] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 129/299 (43% done) --- + [Epoch 129][10/66] loss=0.768595 avg=0.884911 VRAM=39.5GiB | 43.1% done | ETA(epoch): 1079s + [Epoch 129][20/66] loss=0.788476 avg=0.881110 VRAM=39.2GiB | 43.1% done | ETA(epoch): 887s + [Epoch 129][30/66] loss=0.956883 avg=0.877127 VRAM=39.5GiB | 43.2% done | ETA(epoch): 694s + [Epoch 129][40/66] loss=0.810039 avg=0.866716 VRAM=39.2GiB | 43.2% done | ETA(epoch): 502s + [Epoch 129][50/66] loss=0.860589 avg=0.867010 VRAM=39.5GiB | 43.3% done | ETA(epoch): 309s + [Epoch 129][60/66] loss=0.839838 avg=0.870816 VRAM=39.2GiB | 43.3% done | ETA(epoch): 116s + Train loss: 0.873023 (1273.8s) ETA: 3968min + Val loss: 0.838033 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9923 t_0.4-0.6=0.8372 t_0.6-0.8=0.7220 t_0.8-1.0=0.6583] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0129 +[MEM @ epoch 129 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 130/299 (43% done) --- + [Epoch 130][10/66] loss=0.982111 avg=0.875060 VRAM=39.5GiB | 43.4% done | ETA(epoch): 1078s + [Epoch 130][20/66] loss=0.871647 avg=0.867977 VRAM=39.2GiB | 43.4% done | ETA(epoch): 887s + [Epoch 130][30/66] loss=0.813472 avg=0.873412 VRAM=39.5GiB | 43.5% done | ETA(epoch): 694s + [Epoch 130][40/66] loss=0.720650 avg=0.860497 VRAM=39.2GiB | 43.5% done | ETA(epoch): 501s + [Epoch 130][50/66] loss=0.848675 avg=0.864664 VRAM=39.5GiB | 43.6% done | ETA(epoch): 309s + [Epoch 130][60/66] loss=0.921005 avg=0.866228 VRAM=39.2GiB | 43.6% done | ETA(epoch): 116s + Train loss: 0.866891 (1272.8s) ETA: 3944min + Val loss: 0.831769 [t_0.0-0.2=1.0660 t_0.2-0.4=1.0254 t_0.4-0.6=0.8344 t_0.6-0.8=0.7310 t_0.8-1.0=0.6639] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 131/299 (44% done) --- + [Epoch 131][10/66] loss=0.856423 avg=0.878165 VRAM=39.5GiB | 43.7% done | ETA(epoch): 1090s + [Epoch 131][20/66] loss=0.950622 avg=0.870769 VRAM=39.2GiB | 43.8% done | ETA(epoch): 890s + [Epoch 131][30/66] loss=0.908280 avg=0.880383 VRAM=39.5GiB | 43.8% done | ETA(epoch): 696s + [Epoch 131][40/66] loss=0.794339 avg=0.863659 VRAM=39.2GiB | 43.9% done | ETA(epoch): 502s + [Epoch 131][50/66] loss=0.870800 avg=0.860884 VRAM=39.5GiB | 43.9% done | ETA(epoch): 309s + [Epoch 131][60/66] loss=0.906221 avg=0.863910 VRAM=39.2GiB | 44.0% done | ETA(epoch): 116s + Train loss: 0.858314 (1274.6s) ETA: 3920min + Val loss: 0.856696 [t_0.0-0.2=1.0593 t_0.2-0.4=1.0438 t_0.4-0.6=0.8432 t_0.6-0.8=0.6999 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 132/299 (44% done) --- + [Epoch 132][10/66] loss=0.913645 avg=0.878937 VRAM=39.5GiB | 44.1% done | ETA(epoch): 1080s + [Epoch 132][20/66] loss=0.774713 avg=0.869923 VRAM=39.2GiB | 44.1% done | ETA(epoch): 890s + [Epoch 132][30/66] loss=0.713565 avg=0.853715 VRAM=39.5GiB | 44.2% done | ETA(epoch): 696s + [Epoch 132][40/66] loss=0.696057 avg=0.862292 VRAM=39.2GiB | 44.2% done | ETA(epoch): 503s + [Epoch 132][50/66] loss=0.889706 avg=0.864442 VRAM=39.5GiB | 44.3% done | ETA(epoch): 309s + [Epoch 132][60/66] loss=0.847281 avg=0.860033 VRAM=39.2GiB | 44.3% done | ETA(epoch): 116s + Train loss: 0.864588 (1277.3s) ETA: 3896min + Val loss: 0.865632 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9661 t_0.4-0.6=0.8371 t_0.6-0.8=0.6894 t_0.8-1.0=0.6633] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 133/299 (44% done) --- + [Epoch 133][10/66] loss=0.681887 avg=0.872879 VRAM=39.5GiB | 44.4% done | ETA(epoch): 1085s + [Epoch 133][20/66] loss=0.772510 avg=0.846030 VRAM=39.2GiB | 44.4% done | ETA(epoch): 891s + [Epoch 133][30/66] loss=0.759844 avg=0.846277 VRAM=39.5GiB | 44.5% done | ETA(epoch): 697s + [Epoch 133][40/66] loss=0.805686 avg=0.855212 VRAM=39.2GiB | 44.5% done | ETA(epoch): 503s + [Epoch 133][50/66] loss=0.839463 avg=0.855168 VRAM=39.5GiB | 44.6% done | ETA(epoch): 310s + [Epoch 133][60/66] loss=0.857013 avg=0.852651 VRAM=39.2GiB | 44.6% done | ETA(epoch): 116s + Train loss: 0.857558 (1280.3s) ETA: 3872min + Val loss: 0.866589 [t_0.0-0.2=1.0792 t_0.2-0.4=1.0245 t_0.4-0.6=0.8241 t_0.6-0.8=0.7250 t_0.8-1.0=0.6687] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 134/299 (45% done) --- + [Epoch 134][10/66] loss=0.783362 avg=0.856626 VRAM=39.5GiB | 44.7% done | ETA(epoch): 1079s + [Epoch 134][20/66] loss=0.871614 avg=0.866787 VRAM=39.2GiB | 44.8% done | ETA(epoch): 888s + [Epoch 134][30/66] loss=0.739470 avg=0.859680 VRAM=39.5GiB | 44.8% done | ETA(epoch): 695s + [Epoch 134][40/66] loss=0.778982 avg=0.852063 VRAM=39.2GiB | 44.9% done | ETA(epoch): 502s + [Epoch 134][50/66] loss=0.807892 avg=0.849209 VRAM=39.5GiB | 44.9% done | ETA(epoch): 310s + [Epoch 134][60/66] loss=0.768964 avg=0.853920 VRAM=39.2GiB | 45.0% done | ETA(epoch): 116s + Train loss: 0.851782 (1275.9s) ETA: 3848min + Val loss: 0.876073 [t_0.0-0.2=1.0596 t_0.2-0.4=1.0151 t_0.4-0.6=0.8973 t_0.6-0.8=0.7109 t_0.8-1.0=0.6775] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 135/299 (45% done) --- + [Epoch 135][10/66] loss=0.956284 avg=0.873276 VRAM=39.5GiB | 45.1% done | ETA(epoch): 1083s + [Epoch 135][20/66] loss=1.050696 avg=0.866001 VRAM=39.2GiB | 45.1% done | ETA(epoch): 887s + [Epoch 135][30/66] loss=0.778613 avg=0.841036 VRAM=39.5GiB | 45.2% done | ETA(epoch): 694s + [Epoch 135][40/66] loss=0.979183 avg=0.854602 VRAM=39.2GiB | 45.2% done | ETA(epoch): 503s + [Epoch 135][50/66] loss=0.870745 avg=0.856786 VRAM=39.5GiB | 45.3% done | ETA(epoch): 309s + [Epoch 135][60/66] loss=0.903666 avg=0.856229 VRAM=39.2GiB | 45.3% done | ETA(epoch): 116s + Train loss: 0.857556 (1275.2s) ETA: 3825min + Val loss: 0.867318 [t_0.0-0.2=1.0845 t_0.2-0.4=1.0097 t_0.4-0.6=0.8766 t_0.6-0.8=0.6833 t_0.8-1.0=0.6671] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 136/299 (45% done) --- + [Epoch 136][10/66] loss=0.972268 avg=0.842938 VRAM=39.5GiB | 45.4% done | ETA(epoch): 1083s + [Epoch 136][20/66] loss=0.808142 avg=0.854860 VRAM=39.2GiB | 45.4% done | ETA(epoch): 889s + [Epoch 136][30/66] loss=0.822285 avg=0.847721 VRAM=39.5GiB | 45.5% done | ETA(epoch): 698s + [Epoch 136][40/66] loss=0.855578 avg=0.842344 VRAM=39.2GiB | 45.5% done | ETA(epoch): 504s + [Epoch 136][50/66] loss=0.885476 avg=0.841245 VRAM=39.5GiB | 45.6% done | ETA(epoch): 310s + [Epoch 136][60/66] loss=0.883765 avg=0.848757 VRAM=39.2GiB | 45.6% done | ETA(epoch): 116s + Train loss: 0.847977 (1276.4s) ETA: 3801min + Val loss: 0.846424 [t_0.0-0.2=1.0520 t_0.2-0.4=1.0135 t_0.4-0.6=0.8394 t_0.6-0.8=0.7250 t_0.8-1.0=0.6696] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 137/299 (46% done) --- + [Epoch 137][10/66] loss=0.773163 avg=0.866711 VRAM=39.5GiB | 45.7% done | ETA(epoch): 1082s + [Epoch 137][20/66] loss=0.829485 avg=0.855564 VRAM=39.2GiB | 45.8% done | ETA(epoch): 892s + [Epoch 137][30/66] loss=0.841655 avg=0.841054 VRAM=39.5GiB | 45.8% done | ETA(epoch): 698s + [Epoch 137][40/66] loss=0.787735 avg=0.841315 VRAM=39.2GiB | 45.9% done | ETA(epoch): 503s + [Epoch 137][50/66] loss=1.034737 avg=0.849831 VRAM=39.5GiB | 45.9% done | ETA(epoch): 310s + [Epoch 137][60/66] loss=0.881433 avg=0.857541 VRAM=39.2GiB | 46.0% done | ETA(epoch): 116s + Train loss: 0.860874 (1276.3s) ETA: 3777min + Val loss: 0.881536 [t_0.0-0.2=1.0761 t_0.2-0.4=1.0044 t_0.4-0.6=0.8530 t_0.6-0.8=0.7022 t_0.8-1.0=0.6827] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 138/299 (46% done) --- + [Epoch 138][10/66] loss=0.799461 avg=0.840526 VRAM=39.5GiB | 46.1% done | ETA(epoch): 1081s + [Epoch 138][20/66] loss=0.919544 avg=0.869175 VRAM=39.2GiB | 46.1% done | ETA(epoch): 893s + [Epoch 138][30/66] loss=0.745807 avg=0.860623 VRAM=39.5GiB | 46.2% done | ETA(epoch): 698s + [Epoch 138][40/66] loss=0.831829 avg=0.861571 VRAM=39.2GiB | 46.2% done | ETA(epoch): 503s + [Epoch 138][50/66] loss=0.840188 avg=0.858756 VRAM=39.5GiB | 46.3% done | ETA(epoch): 310s + [Epoch 138][60/66] loss=0.840078 avg=0.858500 VRAM=39.2GiB | 46.3% done | ETA(epoch): 116s + Train loss: 0.861974 (1277.3s) ETA: 3753min + Val loss: 0.884639 [t_0.0-0.2=1.0594 t_0.2-0.4=1.0230 t_0.4-0.6=0.8423 t_0.6-0.8=0.7294 t_0.8-1.0=0.6533] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 139/299 (46% done) --- + [Epoch 139][10/66] loss=0.909132 avg=0.852987 VRAM=39.5GiB | 46.4% done | ETA(epoch): 1083s + [Epoch 139][20/66] loss=0.826631 avg=0.839077 VRAM=39.2GiB | 46.4% done | ETA(epoch): 890s + [Epoch 139][30/66] loss=0.926501 avg=0.860508 VRAM=39.5GiB | 46.5% done | ETA(epoch): 696s + [Epoch 139][40/66] loss=0.772112 avg=0.864427 VRAM=39.2GiB | 46.5% done | ETA(epoch): 502s + [Epoch 139][50/66] loss=1.048626 avg=0.864457 VRAM=39.5GiB | 46.6% done | ETA(epoch): 309s + [Epoch 139][60/66] loss=0.830881 avg=0.856201 VRAM=39.2GiB | 46.6% done | ETA(epoch): 116s + Train loss: 0.854851 (1279.0s) ETA: 3729min + Val loss: 0.881456 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9827 t_0.4-0.6=0.8465 t_0.6-0.8=0.7432 t_0.8-1.0=0.6595] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 140/299 (47% done) --- + [Epoch 140][10/66] loss=0.948326 avg=0.873632 VRAM=39.5GiB | 46.7% done | ETA(epoch): 1082s + [Epoch 140][20/66] loss=0.815489 avg=0.870978 VRAM=39.2GiB | 46.8% done | ETA(epoch): 889s + [Epoch 140][30/66] loss=0.919011 avg=0.870002 VRAM=39.5GiB | 46.8% done | ETA(epoch): 695s + [Epoch 140][40/66] loss=0.910490 avg=0.871608 VRAM=39.2GiB | 46.9% done | ETA(epoch): 502s + [Epoch 140][50/66] loss=0.789888 avg=0.860808 VRAM=39.5GiB | 46.9% done | ETA(epoch): 309s + [Epoch 140][60/66] loss=0.878552 avg=0.862371 VRAM=39.2GiB | 47.0% done | ETA(epoch): 116s + Train loss: 0.861235 (1277.1s) ETA: 3705min + Val loss: 0.881250 [t_0.0-0.2=1.0673 t_0.2-0.4=0.9878 t_0.4-0.6=0.8356 t_0.6-0.8=0.7277 t_0.8-1.0=0.6737] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 141/299 (47% done) --- + [Epoch 141][10/66] loss=0.834558 avg=0.852124 VRAM=39.5GiB | 47.1% done | ETA(epoch): 1080s + [Epoch 141][20/66] loss=0.795274 avg=0.839210 VRAM=39.2GiB | 47.1% done | ETA(epoch): 888s + [Epoch 141][30/66] loss=0.941171 avg=0.847676 VRAM=39.5GiB | 47.2% done | ETA(epoch): 695s + [Epoch 141][40/66] loss=0.864207 avg=0.846180 VRAM=39.2GiB | 47.2% done | ETA(epoch): 502s + [Epoch 141][50/66] loss=0.984803 avg=0.847970 VRAM=39.5GiB | 47.3% done | ETA(epoch): 310s + [Epoch 141][60/66] loss=0.879333 avg=0.854574 VRAM=39.2GiB | 47.3% done | ETA(epoch): 116s + Train loss: 0.855694 (1277.3s) ETA: 3682min + Val loss: 0.837229 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9467 t_0.4-0.6=0.8619 t_0.6-0.8=0.7171 t_0.8-1.0=0.6595] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 142/299 (47% done) --- + [Epoch 142][10/66] loss=0.921140 avg=0.885526 VRAM=39.5GiB | 47.4% done | ETA(epoch): 1080s + [Epoch 142][20/66] loss=0.755221 avg=0.883380 VRAM=39.2GiB | 47.4% done | ETA(epoch): 888s + [Epoch 142][30/66] loss=0.809181 avg=0.871546 VRAM=39.5GiB | 47.5% done | ETA(epoch): 696s + [Epoch 142][40/66] loss=0.892880 avg=0.861843 VRAM=39.2GiB | 47.5% done | ETA(epoch): 503s + [Epoch 142][50/66] loss=0.968309 avg=0.859699 VRAM=39.5GiB | 47.6% done | ETA(epoch): 310s + [Epoch 142][60/66] loss=0.834119 avg=0.863606 VRAM=39.2GiB | 47.6% done | ETA(epoch): 116s + Train loss: 0.861326 (1276.6s) ETA: 3658min + Val loss: 0.880464 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9877 t_0.4-0.6=0.8696 t_0.6-0.8=0.6987 t_0.8-1.0=0.6686] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 143/299 (48% done) --- + [Epoch 143][10/66] loss=0.872151 avg=0.894674 VRAM=39.5GiB | 47.7% done | ETA(epoch): 1079s + [Epoch 143][20/66] loss=0.759654 avg=0.853786 VRAM=39.2GiB | 47.8% done | ETA(epoch): 887s + [Epoch 143][30/66] loss=0.783223 avg=0.848475 VRAM=39.5GiB | 47.8% done | ETA(epoch): 694s + [Epoch 143][40/66] loss=0.695160 avg=0.844334 VRAM=39.2GiB | 47.9% done | ETA(epoch): 501s + [Epoch 143][50/66] loss=0.896500 avg=0.844167 VRAM=39.5GiB | 47.9% done | ETA(epoch): 308s + [Epoch 143][60/66] loss=0.939986 avg=0.843665 VRAM=39.2GiB | 48.0% done | ETA(epoch): 116s + Train loss: 0.845908 (1272.6s) ETA: 3634min + Val loss: 0.865977 [t_0.0-0.2=1.0844 t_0.2-0.4=1.0272 t_0.4-0.6=0.8296 t_0.6-0.8=0.7005 t_0.8-1.0=0.6537] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 144/299 (48% done) --- + [Epoch 144][10/66] loss=0.932584 avg=0.842565 VRAM=39.5GiB | 48.1% done | ETA(epoch): 1079s + [Epoch 144][20/66] loss=0.735846 avg=0.845600 VRAM=39.2GiB | 48.1% done | ETA(epoch): 887s + [Epoch 144][30/66] loss=0.857398 avg=0.851076 VRAM=39.5GiB | 48.2% done | ETA(epoch): 697s + [Epoch 144][40/66] loss=0.877304 avg=0.855158 VRAM=39.2GiB | 48.2% done | ETA(epoch): 503s + [Epoch 144][50/66] loss=0.703556 avg=0.851150 VRAM=39.5GiB | 48.3% done | ETA(epoch): 309s + [Epoch 144][60/66] loss=0.773486 avg=0.857581 VRAM=39.2GiB | 48.3% done | ETA(epoch): 116s + Train loss: 0.855249 (1275.2s) ETA: 3610min + Val loss: 0.855841 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9754 t_0.4-0.6=0.8572 t_0.6-0.8=0.7311 t_0.8-1.0=0.6813] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 145/299 (48% done) --- + [Epoch 145][10/66] loss=0.758314 avg=0.844942 VRAM=39.5GiB | 48.4% done | ETA(epoch): 1082s + [Epoch 145][20/66] loss=0.773355 avg=0.840774 VRAM=39.2GiB | 48.4% done | ETA(epoch): 901s + [Epoch 145][30/66] loss=0.987261 avg=0.845098 VRAM=39.5GiB | 48.5% done | ETA(epoch): 701s + [Epoch 145][40/66] loss=0.838130 avg=0.845048 VRAM=39.2GiB | 48.5% done | ETA(epoch): 505s + [Epoch 145][50/66] loss=0.877653 avg=0.844456 VRAM=39.5GiB | 48.6% done | ETA(epoch): 310s + [Epoch 145][60/66] loss=0.893260 avg=0.849010 VRAM=39.2GiB | 48.6% done | ETA(epoch): 116s + Train loss: 0.850305 (1278.1s) ETA: 3587min + Val loss: 0.858829 [t_0.0-0.2=1.0510 t_0.2-0.4=1.0404 t_0.4-0.6=0.8293 t_0.6-0.8=0.7312 t_0.8-1.0=0.6665] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 146/299 (49% done) --- + [Epoch 146][10/66] loss=0.809123 avg=0.819325 VRAM=39.5GiB | 48.7% done | ETA(epoch): 1089s + [Epoch 146][20/66] loss=0.977920 avg=0.832004 VRAM=39.2GiB | 48.8% done | ETA(epoch): 890s + [Epoch 146][30/66] loss=0.879624 avg=0.855880 VRAM=39.5GiB | 48.8% done | ETA(epoch): 696s + [Epoch 146][40/66] loss=0.747436 avg=0.852135 VRAM=39.2GiB | 48.9% done | ETA(epoch): 502s + [Epoch 146][50/66] loss=0.892813 avg=0.865427 VRAM=39.5GiB | 48.9% done | ETA(epoch): 309s + [Epoch 146][60/66] loss=0.861434 avg=0.862983 VRAM=39.2GiB | 49.0% done | ETA(epoch): 116s + Train loss: 0.861580 (1274.3s) ETA: 3563min + Val loss: 0.863335 [t_0.0-0.2=1.0838 t_0.2-0.4=1.0027 t_0.4-0.6=0.8257 t_0.6-0.8=0.7117 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 147/299 (49% done) --- + [Epoch 147][10/66] loss=0.873467 avg=0.873175 VRAM=39.5GiB | 49.1% done | ETA(epoch): 1078s + [Epoch 147][20/66] loss=0.940115 avg=0.847127 VRAM=39.2GiB | 49.1% done | ETA(epoch): 886s + [Epoch 147][30/66] loss=0.965601 avg=0.855169 VRAM=39.5GiB | 49.2% done | ETA(epoch): 694s + [Epoch 147][40/66] loss=0.746469 avg=0.849653 VRAM=39.2GiB | 49.2% done | ETA(epoch): 501s + [Epoch 147][50/66] loss=0.837850 avg=0.858389 VRAM=39.5GiB | 49.3% done | ETA(epoch): 308s + [Epoch 147][60/66] loss=0.863822 avg=0.855087 VRAM=39.2GiB | 49.3% done | ETA(epoch): 116s + Train loss: 0.853124 (1273.1s) ETA: 3539min + Val loss: 0.870754 [t_0.0-0.2=1.0615 t_0.2-0.4=0.9631 t_0.4-0.6=0.8276 t_0.6-0.8=0.7383 t_0.8-1.0=0.6512] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 148/299 (49% done) --- + [Epoch 148][10/66] loss=0.954843 avg=0.846339 VRAM=39.5GiB | 49.4% done | ETA(epoch): 1077s + [Epoch 148][20/66] loss=0.906589 avg=0.846029 VRAM=39.2GiB | 49.4% done | ETA(epoch): 887s + [Epoch 148][30/66] loss=0.986474 avg=0.867628 VRAM=39.5GiB | 49.5% done | ETA(epoch): 694s + [Epoch 148][40/66] loss=0.726151 avg=0.863023 VRAM=39.2GiB | 49.5% done | ETA(epoch): 501s + [Epoch 148][50/66] loss=0.870681 avg=0.862543 VRAM=39.5GiB | 49.6% done | ETA(epoch): 309s + [Epoch 148][60/66] loss=0.914168 avg=0.866103 VRAM=39.2GiB | 49.6% done | ETA(epoch): 116s + Train loss: 0.863920 (1274.7s) ETA: 3515min + Val loss: 0.844883 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9789 t_0.4-0.6=0.8536 t_0.6-0.8=0.7204 t_0.8-1.0=0.6489] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 149/299 (50% done) --- + [Epoch 149][10/66] loss=0.806400 avg=0.865980 VRAM=39.5GiB | 49.7% done | ETA(epoch): 1079s + [Epoch 149][20/66] loss=0.939628 avg=0.854832 VRAM=39.2GiB | 49.8% done | ETA(epoch): 886s + [Epoch 149][30/66] loss=0.697251 avg=0.844908 VRAM=39.5GiB | 49.8% done | ETA(epoch): 694s + [Epoch 149][40/66] loss=0.890876 avg=0.844465 VRAM=39.2GiB | 49.9% done | ETA(epoch): 502s + [Epoch 149][50/66] loss=0.775821 avg=0.854521 VRAM=39.5GiB | 49.9% done | ETA(epoch): 310s + [Epoch 149][60/66] loss=0.863937 avg=0.860536 VRAM=39.2GiB | 50.0% done | ETA(epoch): 116s + Train loss: 0.865569 (1276.6s) ETA: 3492min + Val loss: 0.830043 [t_0.0-0.2=1.0612 t_0.2-0.4=1.0402 t_0.4-0.6=0.8673 t_0.6-0.8=0.7210 t_0.8-1.0=0.6700] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 19.3/188.5 GiB (10.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 150/299 (50% done) --- + [MilestoneVis] train_0 step 9900 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9900 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9900 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9900 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9900 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9900 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9900 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9900 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9900 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9900 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/66] loss=0.778630 avg=0.831501 VRAM=39.5GiB | 50.1% done | ETA(epoch): 6849s + [Epoch 150][20/66] loss=0.860425 avg=0.848690 VRAM=39.2GiB | 50.1% done | ETA(epoch): 3257s + [Epoch 150][30/66] loss=0.938175 avg=0.848760 VRAM=39.5GiB | 50.2% done | ETA(epoch): 1930s + [Epoch 150][40/66] loss=0.916008 avg=0.849101 VRAM=39.2GiB | 50.2% done | ETA(epoch): 1171s + [Epoch 150][50/66] loss=0.807221 avg=0.848208 VRAM=39.5GiB | 50.3% done | ETA(epoch): 638s + [Epoch 150][60/66] loss=0.950926 avg=0.855318 VRAM=39.2GiB | 50.3% done | ETA(epoch): 219s + Train loss: 0.851899 (2303.0s) ETA: 3485min + Val loss: 0.855228 [t_0.0-0.2=1.0704 t_0.2-0.4=0.9507 t_0.4-0.6=0.8385 t_0.6-0.8=0.7381 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 151/299 (50% done) --- + [Epoch 151][10/66] loss=0.799274 avg=0.859924 VRAM=39.5GiB | 50.4% done | ETA(epoch): 1079s + [Epoch 151][20/66] loss=0.910852 avg=0.854579 VRAM=39.2GiB | 50.4% done | ETA(epoch): 886s + [Epoch 151][30/66] loss=0.886232 avg=0.853858 VRAM=39.5GiB | 50.5% done | ETA(epoch): 694s + [Epoch 151][40/66] loss=0.909968 avg=0.853804 VRAM=39.2GiB | 50.5% done | ETA(epoch): 502s + [Epoch 151][50/66] loss=0.904143 avg=0.858218 VRAM=39.5GiB | 50.6% done | ETA(epoch): 309s + [Epoch 151][60/66] loss=0.905054 avg=0.856534 VRAM=39.2GiB | 50.6% done | ETA(epoch): 116s + Train loss: 0.858498 (1274.2s) ETA: 3461min + Val loss: 0.844842 [t_0.0-0.2=1.0638 t_0.2-0.4=0.9966 t_0.4-0.6=0.8563 t_0.6-0.8=0.6992 t_0.8-1.0=0.6580] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 152/299 (51% done) --- + [Epoch 152][10/66] loss=0.730518 avg=0.810986 VRAM=39.5GiB | 50.7% done | ETA(epoch): 1079s + [Epoch 152][20/66] loss=0.874332 avg=0.820079 VRAM=39.2GiB | 50.8% done | ETA(epoch): 888s + [Epoch 152][30/66] loss=0.898629 avg=0.841884 VRAM=39.5GiB | 50.8% done | ETA(epoch): 704s + [Epoch 152][40/66] loss=0.841633 avg=0.848258 VRAM=39.2GiB | 50.9% done | ETA(epoch): 506s + [Epoch 152][50/66] loss=0.975112 avg=0.855719 VRAM=39.5GiB | 50.9% done | ETA(epoch): 311s + [Epoch 152][60/66] loss=0.915393 avg=0.855735 VRAM=39.2GiB | 51.0% done | ETA(epoch): 116s + Train loss: 0.854910 (1279.7s) ETA: 3437min + Val loss: 0.865242 [t_0.0-0.2=1.0543 t_0.2-0.4=0.9754 t_0.4-0.6=0.8126 t_0.6-0.8=0.7217 t_0.8-1.0=0.6519] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 153/299 (51% done) --- + [Epoch 153][10/66] loss=0.914541 avg=0.818266 VRAM=39.5GiB | 51.1% done | ETA(epoch): 1082s + [Epoch 153][20/66] loss=0.826482 avg=0.829095 VRAM=39.2GiB | 51.1% done | ETA(epoch): 893s + [Epoch 153][30/66] loss=0.995474 avg=0.848758 VRAM=39.5GiB | 51.2% done | ETA(epoch): 697s + [Epoch 153][40/66] loss=0.792587 avg=0.847558 VRAM=39.2GiB | 51.2% done | ETA(epoch): 503s + [Epoch 153][50/66] loss=0.808383 avg=0.846528 VRAM=39.5GiB | 51.3% done | ETA(epoch): 309s + [Epoch 153][60/66] loss=0.776271 avg=0.846169 VRAM=39.2GiB | 51.3% done | ETA(epoch): 116s + Train loss: 0.846687 (1274.9s) ETA: 3413min + Val loss: 0.854901 [t_0.0-0.2=1.0813 t_0.2-0.4=0.9696 t_0.4-0.6=0.8573 t_0.6-0.8=0.7021 t_0.8-1.0=0.6551] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 154/299 (51% done) --- + [Epoch 154][10/66] loss=0.831301 avg=0.872071 VRAM=39.5GiB | 51.4% done | ETA(epoch): 1126s + [Epoch 154][20/66] loss=0.793548 avg=0.861047 VRAM=39.2GiB | 51.4% done | ETA(epoch): 906s + [Epoch 154][30/66] loss=0.893453 avg=0.868522 VRAM=39.5GiB | 51.5% done | ETA(epoch): 705s + [Epoch 154][40/66] loss=0.853854 avg=0.867019 VRAM=39.2GiB | 51.5% done | ETA(epoch): 506s + [Epoch 154][50/66] loss=0.796942 avg=0.864641 VRAM=39.5GiB | 51.6% done | ETA(epoch): 311s + [Epoch 154][60/66] loss=0.898436 avg=0.864254 VRAM=39.2GiB | 51.6% done | ETA(epoch): 117s + Train loss: 0.863409 (1280.4s) ETA: 3389min + Val loss: 0.846004 [t_0.0-0.2=1.0759 t_0.2-0.4=0.9690 t_0.4-0.6=0.8477 t_0.6-0.8=0.7254 t_0.8-1.0=0.6446] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 155/299 (52% done) --- + [Epoch 155][10/66] loss=0.770414 avg=0.872965 VRAM=39.5GiB | 51.7% done | ETA(epoch): 1080s + [Epoch 155][20/66] loss=0.842024 avg=0.856529 VRAM=39.2GiB | 51.8% done | ETA(epoch): 886s + [Epoch 155][30/66] loss=0.953017 avg=0.854275 VRAM=39.5GiB | 51.8% done | ETA(epoch): 694s + [Epoch 155][40/66] loss=0.876668 avg=0.847187 VRAM=39.2GiB | 51.9% done | ETA(epoch): 501s + [Epoch 155][50/66] loss=0.855548 avg=0.853050 VRAM=39.5GiB | 51.9% done | ETA(epoch): 309s + [Epoch 155][60/66] loss=0.947690 avg=0.850611 VRAM=39.2GiB | 52.0% done | ETA(epoch): 116s + Train loss: 0.854289 (1274.2s) ETA: 3365min + Val loss: 0.855168 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9855 t_0.4-0.6=0.8191 t_0.6-0.8=0.7375 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 156/299 (52% done) --- + [Epoch 156][10/66] loss=0.851939 avg=0.876286 VRAM=39.5GiB | 52.1% done | ETA(epoch): 1082s + [Epoch 156][20/66] loss=0.878270 avg=0.878747 VRAM=39.2GiB | 52.1% done | ETA(epoch): 888s + [Epoch 156][30/66] loss=0.795990 avg=0.878748 VRAM=39.5GiB | 52.2% done | ETA(epoch): 695s + [Epoch 156][40/66] loss=0.890667 avg=0.869239 VRAM=39.2GiB | 52.2% done | ETA(epoch): 502s + [Epoch 156][50/66] loss=0.865973 avg=0.868478 VRAM=39.5GiB | 52.3% done | ETA(epoch): 309s + [Epoch 156][60/66] loss=0.846962 avg=0.869385 VRAM=39.2GiB | 52.3% done | ETA(epoch): 116s + Train loss: 0.866740 (1274.2s) ETA: 3342min + Val loss: 0.856522 [t_0.0-0.2=1.0843 t_0.2-0.4=1.0042 t_0.4-0.6=0.8594 t_0.6-0.8=0.7089 t_0.8-1.0=0.6670] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 157/299 (52% done) --- + [Epoch 157][10/66] loss=0.795360 avg=0.847956 VRAM=39.5GiB | 52.4% done | ETA(epoch): 1080s + [Epoch 157][20/66] loss=0.838885 avg=0.856438 VRAM=39.2GiB | 52.4% done | ETA(epoch): 887s + [Epoch 157][30/66] loss=0.871387 avg=0.861681 VRAM=39.5GiB | 52.5% done | ETA(epoch): 694s + [Epoch 157][40/66] loss=0.855182 avg=0.856862 VRAM=39.2GiB | 52.5% done | ETA(epoch): 501s + [Epoch 157][50/66] loss=0.919184 avg=0.858731 VRAM=39.5GiB | 52.6% done | ETA(epoch): 308s + [Epoch 157][60/66] loss=0.848511 avg=0.859983 VRAM=39.2GiB | 52.6% done | ETA(epoch): 116s + Train loss: 0.860541 (1272.7s) ETA: 3318min + Val loss: 0.896254 [t_0.0-0.2=1.0708 t_0.2-0.4=0.9929 t_0.4-0.6=0.8224 t_0.6-0.8=0.7544 t_0.8-1.0=0.6629] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 158/299 (53% done) --- + [Epoch 158][10/66] loss=0.849032 avg=0.854977 VRAM=39.5GiB | 52.7% done | ETA(epoch): 1078s + [Epoch 158][20/66] loss=0.860261 avg=0.865397 VRAM=39.2GiB | 52.8% done | ETA(epoch): 887s + [Epoch 158][30/66] loss=0.874038 avg=0.855844 VRAM=39.5GiB | 52.8% done | ETA(epoch): 695s + [Epoch 158][40/66] loss=0.892145 avg=0.850311 VRAM=39.2GiB | 52.9% done | ETA(epoch): 504s + [Epoch 158][50/66] loss=0.843192 avg=0.854615 VRAM=39.5GiB | 52.9% done | ETA(epoch): 310s + [Epoch 158][60/66] loss=0.932098 avg=0.855855 VRAM=39.2GiB | 53.0% done | ETA(epoch): 116s + Train loss: 0.857845 (1279.1s) ETA: 3294min + Val loss: 0.850756 [t_0.0-0.2=1.0594 t_0.2-0.4=0.9843 t_0.4-0.6=0.8517 t_0.6-0.8=0.7447 t_0.8-1.0=0.6575] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 159/299 (53% done) --- + [Epoch 159][10/66] loss=0.871810 avg=0.826843 VRAM=39.5GiB | 53.1% done | ETA(epoch): 1078s + [Epoch 159][20/66] loss=0.879330 avg=0.852141 VRAM=39.2GiB | 53.1% done | ETA(epoch): 885s + [Epoch 159][30/66] loss=0.948244 avg=0.852469 VRAM=39.5GiB | 53.2% done | ETA(epoch): 696s + [Epoch 159][40/66] loss=0.802733 avg=0.846051 VRAM=39.2GiB | 53.2% done | ETA(epoch): 502s + [Epoch 159][50/66] loss=0.769357 avg=0.844753 VRAM=39.5GiB | 53.3% done | ETA(epoch): 309s + [Epoch 159][60/66] loss=0.979194 avg=0.854809 VRAM=39.2GiB | 53.3% done | ETA(epoch): 116s + Train loss: 0.859109 (1273.5s) ETA: 3270min + Val loss: 0.863163 [t_0.0-0.2=1.0594 t_0.2-0.4=1.0069 t_0.4-0.6=0.8700 t_0.6-0.8=0.7365 t_0.8-1.0=0.6583] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 160/299 (53% done) --- + [Epoch 160][10/66] loss=0.916394 avg=0.925148 VRAM=39.5GiB | 53.4% done | ETA(epoch): 1078s + [Epoch 160][20/66] loss=1.005372 avg=0.923369 VRAM=39.2GiB | 53.4% done | ETA(epoch): 899s + [Epoch 160][30/66] loss=0.885385 avg=0.901153 VRAM=39.5GiB | 53.5% done | ETA(epoch): 700s + [Epoch 160][40/66] loss=0.975873 avg=0.895328 VRAM=39.2GiB | 53.5% done | ETA(epoch): 504s + [Epoch 160][50/66] loss=0.879422 avg=0.881705 VRAM=39.5GiB | 53.6% done | ETA(epoch): 310s + [Epoch 160][60/66] loss=0.863483 avg=0.877000 VRAM=39.2GiB | 53.6% done | ETA(epoch): 116s + Train loss: 0.879085 (1278.3s) ETA: 3246min + Val loss: 0.838152 [t_0.0-0.2=1.0849 t_0.2-0.4=0.9541 t_0.4-0.6=0.8661 t_0.6-0.8=0.7163 t_0.8-1.0=0.6532] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 161/299 (54% done) --- + [Epoch 161][10/66] loss=0.796177 avg=0.840403 VRAM=39.5GiB | 53.7% done | ETA(epoch): 1083s + [Epoch 161][20/66] loss=0.793254 avg=0.856916 VRAM=39.2GiB | 53.8% done | ETA(epoch): 894s + [Epoch 161][30/66] loss=0.990181 avg=0.855122 VRAM=39.5GiB | 53.8% done | ETA(epoch): 698s + [Epoch 161][40/66] loss=0.979848 avg=0.855445 VRAM=39.2GiB | 53.9% done | ETA(epoch): 503s + [Epoch 161][50/66] loss=0.732577 avg=0.855104 VRAM=39.5GiB | 53.9% done | ETA(epoch): 309s + [Epoch 161][60/66] loss=0.936613 avg=0.855317 VRAM=39.2GiB | 54.0% done | ETA(epoch): 116s + Train loss: 0.855862 (1274.4s) ETA: 3223min + Val loss: 0.874212 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9830 t_0.4-0.6=0.8407 t_0.6-0.8=0.6917 t_0.8-1.0=0.6597] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 162/299 (54% done) --- + [Epoch 162][10/66] loss=0.849046 avg=0.842394 VRAM=39.5GiB | 54.1% done | ETA(epoch): 1123s + [Epoch 162][20/66] loss=0.872856 avg=0.854692 VRAM=39.2GiB | 54.1% done | ETA(epoch): 903s + [Epoch 162][30/66] loss=0.785653 avg=0.862529 VRAM=39.5GiB | 54.2% done | ETA(epoch): 703s + [Epoch 162][40/66] loss=0.848896 avg=0.859326 VRAM=39.2GiB | 54.2% done | ETA(epoch): 506s + [Epoch 162][50/66] loss=0.746619 avg=0.855432 VRAM=39.5GiB | 54.3% done | ETA(epoch): 311s + [Epoch 162][60/66] loss=0.949275 avg=0.856933 VRAM=39.2GiB | 54.3% done | ETA(epoch): 116s + Train loss: 0.854428 (1281.8s) ETA: 3199min + Val loss: 0.848622 [t_0.0-0.2=1.0735 t_0.2-0.4=1.0277 t_0.4-0.6=0.8825 t_0.6-0.8=0.7218 t_0.8-1.0=0.6657] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 20.1/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 163/299 (54% done) --- + [Epoch 163][10/66] loss=0.791966 avg=0.844087 VRAM=39.5GiB | 54.4% done | ETA(epoch): 1078s + [Epoch 163][20/66] loss=0.815010 avg=0.862237 VRAM=39.2GiB | 54.4% done | ETA(epoch): 887s + [Epoch 163][30/66] loss=0.732685 avg=0.862590 VRAM=39.5GiB | 54.5% done | ETA(epoch): 695s + [Epoch 163][40/66] loss=0.915884 avg=0.862242 VRAM=39.2GiB | 54.5% done | ETA(epoch): 502s + [Epoch 163][50/66] loss=0.926694 avg=0.860585 VRAM=39.5GiB | 54.6% done | ETA(epoch): 309s + [Epoch 163][60/66] loss=0.886595 avg=0.857318 VRAM=39.2GiB | 54.6% done | ETA(epoch): 116s + Train loss: 0.858381 (1273.6s) ETA: 3175min + Val loss: 0.856861 [t_0.0-0.2=1.0779 t_0.2-0.4=1.0039 t_0.4-0.6=0.8666 t_0.6-0.8=0.7111 t_0.8-1.0=0.6535] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 164/299 (55% done) --- + [Epoch 164][10/66] loss=0.854871 avg=0.844127 VRAM=39.5GiB | 54.7% done | ETA(epoch): 1079s + [Epoch 164][20/66] loss=0.908808 avg=0.871808 VRAM=39.2GiB | 54.8% done | ETA(epoch): 886s + [Epoch 164][30/66] loss=0.911677 avg=0.861532 VRAM=39.5GiB | 54.8% done | ETA(epoch): 694s + [Epoch 164][40/66] loss=0.936627 avg=0.861451 VRAM=39.2GiB | 54.9% done | ETA(epoch): 501s + [Epoch 164][50/66] loss=0.880191 avg=0.865418 VRAM=39.5GiB | 54.9% done | ETA(epoch): 309s + [Epoch 164][60/66] loss=0.815067 avg=0.861431 VRAM=39.2GiB | 55.0% done | ETA(epoch): 116s + Train loss: 0.864573 (1273.9s) ETA: 3151min + Val loss: 0.863675 [t_0.0-0.2=1.0609 t_0.2-0.4=0.9687 t_0.4-0.6=0.8474 t_0.6-0.8=0.7157 t_0.8-1.0=0.6611] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 165/299 (55% done) --- + [Epoch 165][10/66] loss=0.944532 avg=0.871655 VRAM=39.5GiB | 55.1% done | ETA(epoch): 1077s + [Epoch 165][20/66] loss=0.939851 avg=0.870419 VRAM=39.2GiB | 55.1% done | ETA(epoch): 886s + [Epoch 165][30/66] loss=0.937698 avg=0.866110 VRAM=39.5GiB | 55.2% done | ETA(epoch): 693s + [Epoch 165][40/66] loss=0.813154 avg=0.867712 VRAM=39.2GiB | 55.2% done | ETA(epoch): 505s + [Epoch 165][50/66] loss=0.830395 avg=0.869281 VRAM=39.5GiB | 55.3% done | ETA(epoch): 311s + [Epoch 165][60/66] loss=0.884968 avg=0.870886 VRAM=39.2GiB | 55.3% done | ETA(epoch): 116s + Train loss: 0.863431 (1278.9s) ETA: 3128min + Val loss: 0.886926 [t_0.0-0.2=1.0656 t_0.2-0.4=1.0122 t_0.4-0.6=0.8260 t_0.6-0.8=0.7213 t_0.8-1.0=0.6750] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 166/299 (55% done) --- + [Epoch 166][10/66] loss=0.918648 avg=0.887880 VRAM=39.5GiB | 55.4% done | ETA(epoch): 1083s + [Epoch 166][20/66] loss=0.891712 avg=0.884279 VRAM=39.2GiB | 55.4% done | ETA(epoch): 888s + [Epoch 166][30/66] loss=0.987359 avg=0.871952 VRAM=39.5GiB | 55.5% done | ETA(epoch): 694s + [Epoch 166][40/66] loss=0.797128 avg=0.860794 VRAM=39.2GiB | 55.5% done | ETA(epoch): 502s + [Epoch 166][50/66] loss=0.868116 avg=0.855074 VRAM=39.5GiB | 55.6% done | ETA(epoch): 309s + [Epoch 166][60/66] loss=0.822474 avg=0.851108 VRAM=39.2GiB | 55.6% done | ETA(epoch): 116s + Train loss: 0.847208 (1273.8s) ETA: 3104min + Val loss: 0.865080 [t_0.0-0.2=1.0642 t_0.2-0.4=0.9848 t_0.4-0.6=0.8209 t_0.6-0.8=0.7373 t_0.8-1.0=0.6551] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 20.1/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 167/299 (56% done) --- + [Epoch 167][10/66] loss=0.869217 avg=0.880879 VRAM=39.5GiB | 55.7% done | ETA(epoch): 1078s + [Epoch 167][20/66] loss=0.886189 avg=0.848890 VRAM=39.2GiB | 55.8% done | ETA(epoch): 888s + [Epoch 167][30/66] loss=0.693814 avg=0.848032 VRAM=39.5GiB | 55.8% done | ETA(epoch): 698s + [Epoch 167][40/66] loss=0.917087 avg=0.852319 VRAM=39.2GiB | 55.9% done | ETA(epoch): 504s + [Epoch 167][50/66] loss=0.817557 avg=0.845943 VRAM=39.5GiB | 55.9% done | ETA(epoch): 310s + [Epoch 167][60/66] loss=0.835248 avg=0.846926 VRAM=39.2GiB | 56.0% done | ETA(epoch): 116s + Train loss: 0.846576 (1277.1s) ETA: 3080min + Val loss: 0.856895 [t_0.0-0.2=1.0708 t_0.2-0.4=1.0026 t_0.4-0.6=0.8426 t_0.6-0.8=0.7247 t_0.8-1.0=0.6633] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 168/299 (56% done) --- + [Epoch 168][10/66] loss=0.727117 avg=0.838744 VRAM=39.5GiB | 56.1% done | ETA(epoch): 1084s + [Epoch 168][20/66] loss=0.834997 avg=0.848831 VRAM=39.2GiB | 56.1% done | ETA(epoch): 892s + [Epoch 168][30/66] loss=0.830952 avg=0.853267 VRAM=39.5GiB | 56.2% done | ETA(epoch): 698s + [Epoch 168][40/66] loss=0.861332 avg=0.859805 VRAM=39.2GiB | 56.2% done | ETA(epoch): 503s + [Epoch 168][50/66] loss=0.906119 avg=0.860708 VRAM=39.5GiB | 56.3% done | ETA(epoch): 309s + [Epoch 168][60/66] loss=0.866836 avg=0.860038 VRAM=39.2GiB | 56.3% done | ETA(epoch): 116s + Train loss: 0.862079 (1275.1s) ETA: 3056min + Val loss: 0.837191 [t_0.0-0.2=1.0836 t_0.2-0.4=1.0130 t_0.4-0.6=0.8201 t_0.6-0.8=0.6982 t_0.8-1.0=0.6472] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 169/299 (56% done) --- + [Epoch 169][10/66] loss=0.776861 avg=0.804938 VRAM=39.5GiB | 56.4% done | ETA(epoch): 1080s + [Epoch 169][20/66] loss=0.845551 avg=0.824763 VRAM=39.2GiB | 56.4% done | ETA(epoch): 888s + [Epoch 169][30/66] loss=0.961365 avg=0.839426 VRAM=39.5GiB | 56.5% done | ETA(epoch): 696s + [Epoch 169][40/66] loss=0.807083 avg=0.832634 VRAM=39.2GiB | 56.5% done | ETA(epoch): 502s + [Epoch 169][50/66] loss=0.753050 avg=0.833656 VRAM=39.5GiB | 56.6% done | ETA(epoch): 309s + [Epoch 169][60/66] loss=0.866767 avg=0.835540 VRAM=39.2GiB | 56.6% done | ETA(epoch): 116s + Train loss: 0.840346 (1274.0s) ETA: 3033min + Val loss: 0.856942 [t_0.0-0.2=1.0773 t_0.2-0.4=1.0513 t_0.4-0.6=0.8275 t_0.6-0.8=0.7106 t_0.8-1.0=0.6584] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 170/299 (57% done) --- + [Epoch 170][10/66] loss=0.971867 avg=0.830747 VRAM=39.5GiB | 56.7% done | ETA(epoch): 1081s + [Epoch 170][20/66] loss=0.978714 avg=0.872566 VRAM=39.2GiB | 56.8% done | ETA(epoch): 887s + [Epoch 170][30/66] loss=0.871965 avg=0.865995 VRAM=39.5GiB | 56.8% done | ETA(epoch): 694s + [Epoch 170][40/66] loss=0.815896 avg=0.862515 VRAM=39.2GiB | 56.9% done | ETA(epoch): 501s + [Epoch 170][50/66] loss=0.849271 avg=0.866503 VRAM=39.5GiB | 56.9% done | ETA(epoch): 309s + [Epoch 170][60/66] loss=0.910331 avg=0.869275 VRAM=39.2GiB | 57.0% done | ETA(epoch): 116s + Train loss: 0.871340 (1273.3s) ETA: 3009min + Val loss: 0.844092 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9861 t_0.4-0.6=0.8228 t_0.6-0.8=0.7347 t_0.8-1.0=0.6593] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 171/299 (57% done) --- + [Epoch 171][10/66] loss=0.907126 avg=0.828219 VRAM=39.5GiB | 57.1% done | ETA(epoch): 1078s + [Epoch 171][20/66] loss=0.840119 avg=0.850491 VRAM=39.2GiB | 57.1% done | ETA(epoch): 887s + [Epoch 171][30/66] loss=0.723849 avg=0.854744 VRAM=39.5GiB | 57.2% done | ETA(epoch): 694s + [Epoch 171][40/66] loss=0.819950 avg=0.858686 VRAM=39.2GiB | 57.2% done | ETA(epoch): 501s + [Epoch 171][50/66] loss=0.788155 avg=0.853884 VRAM=39.5GiB | 57.3% done | ETA(epoch): 310s + [Epoch 171][60/66] loss=0.798845 avg=0.854542 VRAM=39.2GiB | 57.3% done | ETA(epoch): 116s + Train loss: 0.854418 (1275.6s) ETA: 2985min + Val loss: 0.846081 [t_0.0-0.2=1.0691 t_0.2-0.4=0.9891 t_0.4-0.6=0.8327 t_0.6-0.8=0.6946 t_0.8-1.0=0.6573] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 172/299 (57% done) --- + [Epoch 172][10/66] loss=0.860156 avg=0.863741 VRAM=39.5GiB | 57.4% done | ETA(epoch): 1078s + [Epoch 172][20/66] loss=0.801774 avg=0.847342 VRAM=39.2GiB | 57.4% done | ETA(epoch): 886s + [Epoch 172][30/66] loss=0.923196 avg=0.861572 VRAM=39.5GiB | 57.5% done | ETA(epoch): 694s + [Epoch 172][40/66] loss=0.834271 avg=0.857804 VRAM=39.2GiB | 57.5% done | ETA(epoch): 501s + [Epoch 172][50/66] loss=0.767402 avg=0.865718 VRAM=39.5GiB | 57.6% done | ETA(epoch): 309s + [Epoch 172][60/66] loss=0.824909 avg=0.861608 VRAM=39.2GiB | 57.6% done | ETA(epoch): 116s + Train loss: 0.859793 (1274.4s) ETA: 2962min + Val loss: 0.832786 [t_0.0-0.2=1.0826 t_0.2-0.4=0.9678 t_0.4-0.6=0.8307 t_0.6-0.8=0.7073 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 19.9/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 173/299 (58% done) --- + [Epoch 173][10/66] loss=0.910779 avg=0.865175 VRAM=39.5GiB | 57.7% done | ETA(epoch): 1081s + [Epoch 173][20/66] loss=0.742886 avg=0.838361 VRAM=39.2GiB | 57.8% done | ETA(epoch): 889s + [Epoch 173][30/66] loss=0.792387 avg=0.848019 VRAM=39.5GiB | 57.8% done | ETA(epoch): 696s + [Epoch 173][40/66] loss=0.793749 avg=0.852329 VRAM=39.2GiB | 57.9% done | ETA(epoch): 503s + [Epoch 173][50/66] loss=0.753739 avg=0.844908 VRAM=39.5GiB | 57.9% done | ETA(epoch): 309s + [Epoch 173][60/66] loss=0.857737 avg=0.851127 VRAM=39.2GiB | 58.0% done | ETA(epoch): 116s + Train loss: 0.855322 (1274.3s) ETA: 2938min + Val loss: 0.885680 [t_0.0-0.2=1.0836 t_0.2-0.4=0.9805 t_0.4-0.6=0.8448 t_0.6-0.8=0.7265 t_0.8-1.0=0.6765] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 174/299 (58% done) --- + [Epoch 174][10/66] loss=0.748703 avg=0.833600 VRAM=39.5GiB | 58.1% done | ETA(epoch): 1080s + [Epoch 174][20/66] loss=0.872230 avg=0.847286 VRAM=39.2GiB | 58.1% done | ETA(epoch): 887s + [Epoch 174][30/66] loss=0.958490 avg=0.848187 VRAM=39.5GiB | 58.2% done | ETA(epoch): 694s + [Epoch 174][40/66] loss=0.870606 avg=0.850073 VRAM=39.2GiB | 58.2% done | ETA(epoch): 501s + [Epoch 174][50/66] loss=0.991394 avg=0.850118 VRAM=39.5GiB | 58.3% done | ETA(epoch): 308s + [Epoch 174][60/66] loss=0.835419 avg=0.858835 VRAM=39.2GiB | 58.3% done | ETA(epoch): 116s + Train loss: 0.855873 (1271.5s) ETA: 2914min + Val loss: 0.834543 [t_0.0-0.2=1.0508 t_0.2-0.4=0.9863 t_0.4-0.6=0.8158 t_0.6-0.8=0.7223 t_0.8-1.0=0.6467] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 175/299 (58% done) --- + [Epoch 175][10/66] loss=0.833805 avg=0.895961 VRAM=39.5GiB | 58.4% done | ETA(epoch): 1078s + [Epoch 175][20/66] loss=0.815936 avg=0.877096 VRAM=39.2GiB | 58.4% done | ETA(epoch): 886s + [Epoch 175][30/66] loss=0.884561 avg=0.888285 VRAM=39.5GiB | 58.5% done | ETA(epoch): 693s + [Epoch 175][40/66] loss=0.700912 avg=0.872477 VRAM=39.2GiB | 58.5% done | ETA(epoch): 501s + [Epoch 175][50/66] loss=0.846351 avg=0.872911 VRAM=39.5GiB | 58.6% done | ETA(epoch): 308s + [Epoch 175][60/66] loss=0.895100 avg=0.870319 VRAM=39.2GiB | 58.6% done | ETA(epoch): 116s + Train loss: 0.869850 (1271.6s) ETA: 2891min + Val loss: 0.893954 [t_0.0-0.2=1.0511 t_0.2-0.4=1.0093 t_0.4-0.6=0.8342 t_0.6-0.8=0.7440 t_0.8-1.0=0.6679] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 176/299 (59% done) --- + [Epoch 176][10/66] loss=0.727182 avg=0.880559 VRAM=39.5GiB | 58.7% done | ETA(epoch): 1092s + [Epoch 176][20/66] loss=0.701645 avg=0.872386 VRAM=39.2GiB | 58.8% done | ETA(epoch): 891s + [Epoch 176][30/66] loss=0.853995 avg=0.880123 VRAM=39.5GiB | 58.8% done | ETA(epoch): 696s + [Epoch 176][40/66] loss=0.832995 avg=0.877578 VRAM=39.2GiB | 58.9% done | ETA(epoch): 503s + [Epoch 176][50/66] loss=0.797301 avg=0.864493 VRAM=39.5GiB | 58.9% done | ETA(epoch): 309s + [Epoch 176][60/66] loss=0.797240 avg=0.857375 VRAM=39.2GiB | 59.0% done | ETA(epoch): 116s + Train loss: 0.859361 (1275.4s) ETA: 2867min + Val loss: 0.863856 [t_0.0-0.2=1.0835 t_0.2-0.4=0.9761 t_0.4-0.6=0.8618 t_0.6-0.8=0.7160 t_0.8-1.0=0.6606] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 177/299 (59% done) --- + [Epoch 177][10/66] loss=0.847237 avg=0.861266 VRAM=39.5GiB | 59.1% done | ETA(epoch): 1081s + [Epoch 177][20/66] loss=0.899914 avg=0.857238 VRAM=39.2GiB | 59.1% done | ETA(epoch): 889s + [Epoch 177][30/66] loss=0.817344 avg=0.857365 VRAM=39.5GiB | 59.2% done | ETA(epoch): 695s + [Epoch 177][40/66] loss=0.877908 avg=0.863978 VRAM=39.2GiB | 59.2% done | ETA(epoch): 502s + [Epoch 177][50/66] loss=0.928448 avg=0.860102 VRAM=39.5GiB | 59.3% done | ETA(epoch): 309s + [Epoch 177][60/66] loss=0.744350 avg=0.855749 VRAM=39.2GiB | 59.3% done | ETA(epoch): 116s + Train loss: 0.857166 (1276.2s) ETA: 2843min + Val loss: 0.840371 [t_0.0-0.2=1.0628 t_0.2-0.4=0.9744 t_0.4-0.6=0.8609 t_0.6-0.8=0.7201 t_0.8-1.0=0.6620] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 19.9/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 178/299 (59% done) --- + [Epoch 178][10/66] loss=0.867896 avg=0.862708 VRAM=39.5GiB | 59.4% done | ETA(epoch): 1081s + [Epoch 178][20/66] loss=0.954949 avg=0.863335 VRAM=39.2GiB | 59.4% done | ETA(epoch): 888s + [Epoch 178][30/66] loss=0.819930 avg=0.848101 VRAM=39.5GiB | 59.5% done | ETA(epoch): 695s + [Epoch 178][40/66] loss=0.767051 avg=0.855065 VRAM=39.2GiB | 59.5% done | ETA(epoch): 502s + [Epoch 178][50/66] loss=0.797704 avg=0.857353 VRAM=39.5GiB | 59.6% done | ETA(epoch): 309s + [Epoch 178][60/66] loss=0.857304 avg=0.852554 VRAM=39.2GiB | 59.6% done | ETA(epoch): 116s + Train loss: 0.850790 (1274.6s) ETA: 2820min + Val loss: 0.866443 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9774 t_0.4-0.6=0.8445 t_0.6-0.8=0.7264 t_0.8-1.0=0.6529] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 179/299 (60% done) --- + [Epoch 179][10/66] loss=0.886391 avg=0.854114 VRAM=39.5GiB | 59.7% done | ETA(epoch): 1083s + [Epoch 179][20/66] loss=0.838962 avg=0.842707 VRAM=39.2GiB | 59.8% done | ETA(epoch): 889s + [Epoch 179][30/66] loss=0.798473 avg=0.842519 VRAM=39.5GiB | 59.8% done | ETA(epoch): 696s + [Epoch 179][40/66] loss=0.801653 avg=0.847295 VRAM=39.2GiB | 59.9% done | ETA(epoch): 502s + [Epoch 179][50/66] loss=0.715288 avg=0.851911 VRAM=39.5GiB | 59.9% done | ETA(epoch): 309s + [Epoch 179][60/66] loss=0.899806 avg=0.856268 VRAM=39.2GiB | 60.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 11879 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 11879 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 11879 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 11879 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 11879 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 11879 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 11879 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 11879 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 11879 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 11879 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + Train loss: 0.857508 (2306.4s) ETA: 2807min + Val loss: 0.874155 [t_0.0-0.2=1.0652 t_0.2-0.4=0.9938 t_0.4-0.6=0.8491 t_0.6-0.8=0.7219 t_0.8-1.0=0.6581] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 180/299 (60% done) --- + [Epoch 180][10/66] loss=0.854156 avg=0.812967 VRAM=39.5GiB | 60.1% done | ETA(epoch): 1079s + [Epoch 180][20/66] loss=0.902489 avg=0.832456 VRAM=39.2GiB | 60.1% done | ETA(epoch): 888s + [Epoch 180][30/66] loss=0.857227 avg=0.831398 VRAM=39.5GiB | 60.2% done | ETA(epoch): 695s + [Epoch 180][40/66] loss=0.969949 avg=0.843658 VRAM=39.2GiB | 60.2% done | ETA(epoch): 502s + [Epoch 180][50/66] loss=0.869951 avg=0.845774 VRAM=39.5GiB | 60.3% done | ETA(epoch): 309s + [Epoch 180][60/66] loss=0.884901 avg=0.848483 VRAM=39.2GiB | 60.3% done | ETA(epoch): 116s + Train loss: 0.850539 (1274.5s) ETA: 2784min + Val loss: 0.823957 [t_0.0-0.2=1.0612 t_0.2-0.4=1.0035 t_0.4-0.6=0.8398 t_0.6-0.8=0.7263 t_0.8-1.0=0.6562] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 181/299 (60% done) --- + [Epoch 181][10/66] loss=0.860252 avg=0.846423 VRAM=39.5GiB | 60.4% done | ETA(epoch): 1082s + [Epoch 181][20/66] loss=0.898395 avg=0.844267 VRAM=39.2GiB | 60.4% done | ETA(epoch): 889s + [Epoch 181][30/66] loss=0.848804 avg=0.847794 VRAM=39.5GiB | 60.5% done | ETA(epoch): 695s + [Epoch 181][40/66] loss=0.871466 avg=0.845075 VRAM=39.2GiB | 60.5% done | ETA(epoch): 504s + [Epoch 181][50/66] loss=0.824789 avg=0.846454 VRAM=39.5GiB | 60.6% done | ETA(epoch): 310s + [Epoch 181][60/66] loss=0.882330 avg=0.846100 VRAM=39.2GiB | 60.6% done | ETA(epoch): 116s + Train loss: 0.848594 (1277.8s) ETA: 2760min + Val loss: 0.865253 [t_0.0-0.2=1.0850 t_0.2-0.4=1.0160 t_0.4-0.6=0.8525 t_0.6-0.8=0.7121 t_0.8-1.0=0.6673] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 182/299 (61% done) --- + [Epoch 182][10/66] loss=0.847156 avg=0.857783 VRAM=39.5GiB | 60.7% done | ETA(epoch): 1082s + [Epoch 182][20/66] loss=0.817413 avg=0.824920 VRAM=39.2GiB | 60.8% done | ETA(epoch): 889s + [Epoch 182][30/66] loss=0.685912 avg=0.836646 VRAM=39.5GiB | 60.8% done | ETA(epoch): 699s + [Epoch 182][40/66] loss=0.838426 avg=0.827986 VRAM=39.2GiB | 60.9% done | ETA(epoch): 504s + [Epoch 182][50/66] loss=0.868972 avg=0.837323 VRAM=39.5GiB | 60.9% done | ETA(epoch): 310s + [Epoch 182][60/66] loss=1.007852 avg=0.838952 VRAM=39.2GiB | 61.0% done | ETA(epoch): 116s + Train loss: 0.842278 (1277.7s) ETA: 2736min + Val loss: 0.850092 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9814 t_0.4-0.6=0.8775 t_0.6-0.8=0.7052 t_0.8-1.0=0.6687] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 183/299 (61% done) --- + [Epoch 183][10/66] loss=0.756584 avg=0.868497 VRAM=39.5GiB | 61.1% done | ETA(epoch): 1082s + [Epoch 183][20/66] loss=0.915282 avg=0.869385 VRAM=39.2GiB | 61.1% done | ETA(epoch): 893s + [Epoch 183][30/66] loss=0.727542 avg=0.860706 VRAM=39.5GiB | 61.2% done | ETA(epoch): 698s + [Epoch 183][40/66] loss=0.938255 avg=0.864167 VRAM=39.2GiB | 61.2% done | ETA(epoch): 504s + [Epoch 183][50/66] loss=0.801752 avg=0.867963 VRAM=39.5GiB | 61.3% done | ETA(epoch): 310s + [Epoch 183][60/66] loss=0.828828 avg=0.862995 VRAM=39.2GiB | 61.3% done | ETA(epoch): 116s + Train loss: 0.865455 (1278.4s) ETA: 2713min + Val loss: 0.874525 [t_0.0-0.2=1.0652 t_0.2-0.4=0.9690 t_0.4-0.6=0.8408 t_0.6-0.8=0.7395 t_0.8-1.0=0.6566] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 184/299 (61% done) --- + [Epoch 184][10/66] loss=1.007314 avg=0.897582 VRAM=39.5GiB | 61.4% done | ETA(epoch): 1082s + [Epoch 184][20/66] loss=1.021137 avg=0.880638 VRAM=39.2GiB | 61.4% done | ETA(epoch): 899s + [Epoch 184][30/66] loss=0.941291 avg=0.877127 VRAM=39.5GiB | 61.5% done | ETA(epoch): 701s + [Epoch 184][40/66] loss=0.842819 avg=0.872200 VRAM=39.2GiB | 61.5% done | ETA(epoch): 505s + [Epoch 184][50/66] loss=0.782536 avg=0.866582 VRAM=39.5GiB | 61.6% done | ETA(epoch): 311s + [Epoch 184][60/66] loss=0.899455 avg=0.863090 VRAM=39.2GiB | 61.6% done | ETA(epoch): 116s + Train loss: 0.864473 (1279.4s) ETA: 2689min + Val loss: 0.893513 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9694 t_0.4-0.6=0.8642 t_0.6-0.8=0.7166 t_0.8-1.0=0.6522] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 185/299 (62% done) --- + [Epoch 185][10/66] loss=0.912262 avg=0.853717 VRAM=39.5GiB | 61.7% done | ETA(epoch): 1081s + [Epoch 185][20/66] loss=0.885224 avg=0.847516 VRAM=39.2GiB | 61.8% done | ETA(epoch): 888s + [Epoch 185][30/66] loss=0.755262 avg=0.852891 VRAM=39.5GiB | 61.8% done | ETA(epoch): 695s + [Epoch 185][40/66] loss=0.795565 avg=0.855149 VRAM=39.2GiB | 61.9% done | ETA(epoch): 503s + [Epoch 185][50/66] loss=0.830697 avg=0.851622 VRAM=39.5GiB | 61.9% done | ETA(epoch): 309s + [Epoch 185][60/66] loss=0.718964 avg=0.848170 VRAM=39.2GiB | 62.0% done | ETA(epoch): 116s + Train loss: 0.851223 (1275.4s) ETA: 2665min + Val loss: 0.827506 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9903 t_0.4-0.6=0.8218 t_0.6-0.8=0.7108 t_0.8-1.0=0.6531] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 186/299 (62% done) --- + [Epoch 186][10/66] loss=0.817392 avg=0.853972 VRAM=39.5GiB | 62.1% done | ETA(epoch): 1083s + [Epoch 186][20/66] loss=0.700367 avg=0.835468 VRAM=39.2GiB | 62.1% done | ETA(epoch): 889s + [Epoch 186][30/66] loss=0.870496 avg=0.834003 VRAM=39.5GiB | 62.2% done | ETA(epoch): 696s + [Epoch 186][40/66] loss=0.784997 avg=0.839437 VRAM=39.2GiB | 62.2% done | ETA(epoch): 503s + [Epoch 186][50/66] loss=0.839320 avg=0.848396 VRAM=39.5GiB | 62.3% done | ETA(epoch): 310s + [Epoch 186][60/66] loss=0.823901 avg=0.855914 VRAM=39.2GiB | 62.3% done | ETA(epoch): 117s + Train loss: 0.854612 (1284.3s) ETA: 2642min + Val loss: 0.830007 [t_0.0-0.2=1.0512 t_0.2-0.4=1.0152 t_0.4-0.6=0.8682 t_0.6-0.8=0.7287 t_0.8-1.0=0.6555] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 187/299 (62% done) --- + [Epoch 187][10/66] loss=0.938483 avg=0.871200 VRAM=39.5GiB | 62.4% done | ETA(epoch): 1082s + [Epoch 187][20/66] loss=0.879520 avg=0.857127 VRAM=39.2GiB | 62.4% done | ETA(epoch): 891s + [Epoch 187][30/66] loss=0.895337 avg=0.851923 VRAM=39.5GiB | 62.5% done | ETA(epoch): 697s + [Epoch 187][40/66] loss=0.837831 avg=0.857690 VRAM=39.2GiB | 62.5% done | ETA(epoch): 503s + [Epoch 187][50/66] loss=0.988006 avg=0.860554 VRAM=39.5GiB | 62.6% done | ETA(epoch): 310s + [Epoch 187][60/66] loss=0.900814 avg=0.865037 VRAM=39.2GiB | 62.6% done | ETA(epoch): 116s + Train loss: 0.861031 (1276.9s) ETA: 2618min + Val loss: 0.860560 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9981 t_0.4-0.6=0.8482 t_0.6-0.8=0.6955 t_0.8-1.0=0.6536] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 188/299 (63% done) --- + [Epoch 188][10/66] loss=0.777109 avg=0.843945 VRAM=39.5GiB | 62.7% done | ETA(epoch): 1081s + [Epoch 188][20/66] loss=0.880805 avg=0.861194 VRAM=39.2GiB | 62.8% done | ETA(epoch): 890s + [Epoch 188][30/66] loss=0.734508 avg=0.859064 VRAM=39.5GiB | 62.8% done | ETA(epoch): 696s + [Epoch 188][40/66] loss=0.882934 avg=0.860726 VRAM=39.2GiB | 62.9% done | ETA(epoch): 504s + [Epoch 188][50/66] loss=0.932027 avg=0.857471 VRAM=39.5GiB | 62.9% done | ETA(epoch): 310s + [Epoch 188][60/66] loss=0.844440 avg=0.859893 VRAM=39.2GiB | 63.0% done | ETA(epoch): 116s + Train loss: 0.855045 (1278.4s) ETA: 2594min + Val loss: 0.846025 [t_0.0-0.2=1.0677 t_0.2-0.4=1.0047 t_0.4-0.6=0.8082 t_0.6-0.8=0.7181 t_0.8-1.0=0.6650] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 189/299 (63% done) --- + [Epoch 189][10/66] loss=0.742730 avg=0.869225 VRAM=39.5GiB | 63.1% done | ETA(epoch): 1084s + [Epoch 189][20/66] loss=0.842710 avg=0.860943 VRAM=39.2GiB | 63.1% done | ETA(epoch): 890s + [Epoch 189][30/66] loss=0.768821 avg=0.851630 VRAM=39.5GiB | 63.2% done | ETA(epoch): 697s + [Epoch 189][40/66] loss=0.816219 avg=0.856924 VRAM=39.2GiB | 63.2% done | ETA(epoch): 503s + [Epoch 189][50/66] loss=0.818543 avg=0.857831 VRAM=39.5GiB | 63.3% done | ETA(epoch): 310s + [Epoch 189][60/66] loss=0.791514 avg=0.857724 VRAM=39.2GiB | 63.3% done | ETA(epoch): 116s + Train loss: 0.853641 (1278.3s) ETA: 2571min + Val loss: 0.869601 [t_0.0-0.2=1.0720 t_0.2-0.4=0.9723 t_0.4-0.6=0.8136 t_0.6-0.8=0.7268 t_0.8-1.0=0.6655] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 190/299 (63% done) --- + [Epoch 190][10/66] loss=0.897758 avg=0.884102 VRAM=39.5GiB | 63.4% done | ETA(epoch): 1080s + [Epoch 190][20/66] loss=0.790657 avg=0.845968 VRAM=39.2GiB | 63.4% done | ETA(epoch): 888s + [Epoch 190][30/66] loss=0.958070 avg=0.859059 VRAM=39.5GiB | 63.5% done | ETA(epoch): 697s + [Epoch 190][40/66] loss=0.770264 avg=0.850481 VRAM=39.2GiB | 63.5% done | ETA(epoch): 503s + [Epoch 190][50/66] loss=0.902105 avg=0.849998 VRAM=39.5GiB | 63.6% done | ETA(epoch): 310s + [Epoch 190][60/66] loss=0.867797 avg=0.845561 VRAM=39.2GiB | 63.6% done | ETA(epoch): 116s + Train loss: 0.843268 (1277.5s) ETA: 2547min + Val loss: 0.823587 [t_0.0-0.2=1.0812 t_0.2-0.4=0.9546 t_0.4-0.6=0.8576 t_0.6-0.8=0.7026 t_0.8-1.0=0.6565] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 191/299 (64% done) --- + [Epoch 191][10/66] loss=0.719510 avg=0.833453 VRAM=39.5GiB | 63.7% done | ETA(epoch): 1081s + [Epoch 191][20/66] loss=0.866992 avg=0.840617 VRAM=39.2GiB | 63.8% done | ETA(epoch): 889s + [Epoch 191][30/66] loss=0.915981 avg=0.853526 VRAM=39.5GiB | 63.8% done | ETA(epoch): 696s + [Epoch 191][40/66] loss=0.874498 avg=0.870448 VRAM=39.2GiB | 63.9% done | ETA(epoch): 503s + [Epoch 191][50/66] loss=0.871706 avg=0.873030 VRAM=39.5GiB | 63.9% done | ETA(epoch): 309s + [Epoch 191][60/66] loss=0.802912 avg=0.868176 VRAM=39.2GiB | 64.0% done | ETA(epoch): 116s + Train loss: 0.863020 (1276.4s) ETA: 2523min + Val loss: 0.821209 [t_0.0-0.2=1.0837 t_0.2-0.4=0.9744 t_0.4-0.6=0.8074 t_0.6-0.8=0.7296 t_0.8-1.0=0.6427] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0191 + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 192/299 (64% done) --- + [Epoch 192][10/66] loss=0.771049 avg=0.831815 VRAM=39.5GiB | 64.1% done | ETA(epoch): 1098s + [Epoch 192][20/66] loss=0.770628 avg=0.848141 VRAM=39.2GiB | 64.1% done | ETA(epoch): 896s + [Epoch 192][30/66] loss=0.911609 avg=0.835039 VRAM=39.5GiB | 64.2% done | ETA(epoch): 699s + [Epoch 192][40/66] loss=0.901667 avg=0.838449 VRAM=39.2GiB | 64.2% done | ETA(epoch): 505s + [Epoch 192][50/66] loss=0.898649 avg=0.839357 VRAM=39.5GiB | 64.3% done | ETA(epoch): 310s + [Epoch 192][60/66] loss=0.899348 avg=0.834875 VRAM=39.2GiB | 64.3% done | ETA(epoch): 116s + Train loss: 0.834852 (1281.9s) ETA: 2500min + Val loss: 0.878108 [t_0.0-0.2=1.0562 t_0.2-0.4=0.9979 t_0.4-0.6=0.8353 t_0.6-0.8=0.7341 t_0.8-1.0=0.6593] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 193/299 (64% done) --- + [Epoch 193][10/66] loss=0.924203 avg=0.870060 VRAM=39.5GiB | 64.4% done | ETA(epoch): 1085s + [Epoch 193][20/66] loss=0.864952 avg=0.865848 VRAM=39.2GiB | 64.4% done | ETA(epoch): 892s + [Epoch 193][30/66] loss=0.795082 avg=0.871590 VRAM=39.5GiB | 64.5% done | ETA(epoch): 698s + [Epoch 193][40/66] loss=0.672333 avg=0.860810 VRAM=39.2GiB | 64.5% done | ETA(epoch): 504s + [Epoch 193][50/66] loss=0.841638 avg=0.865068 VRAM=39.5GiB | 64.6% done | ETA(epoch): 310s + [Epoch 193][60/66] loss=0.926759 avg=0.860041 VRAM=39.2GiB | 64.6% done | ETA(epoch): 116s + Train loss: 0.858028 (1277.9s) ETA: 2476min + Val loss: 0.882735 [t_0.0-0.2=1.0721 t_0.2-0.4=0.9767 t_0.4-0.6=0.8532 t_0.6-0.8=0.7291 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 194/299 (65% done) --- + [Epoch 194][10/66] loss=0.843258 avg=0.849505 VRAM=39.5GiB | 64.7% done | ETA(epoch): 1083s + [Epoch 194][20/66] loss=0.962778 avg=0.861355 VRAM=39.2GiB | 64.8% done | ETA(epoch): 890s + [Epoch 194][30/66] loss=0.748132 avg=0.858601 VRAM=39.5GiB | 64.8% done | ETA(epoch): 696s + [Epoch 194][40/66] loss=0.867047 avg=0.855334 VRAM=39.2GiB | 64.9% done | ETA(epoch): 503s + [Epoch 194][50/66] loss=0.703263 avg=0.852294 VRAM=39.5GiB | 64.9% done | ETA(epoch): 310s + [Epoch 194][60/66] loss=0.784002 avg=0.848513 VRAM=39.2GiB | 65.0% done | ETA(epoch): 116s + Train loss: 0.846279 (1279.5s) ETA: 2452min + Val loss: 0.863639 [t_0.0-0.2=1.0563 t_0.2-0.4=1.0134 t_0.4-0.6=0.8332 t_0.6-0.8=0.7268 t_0.8-1.0=0.6514] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 195/299 (65% done) --- + [Epoch 195][10/66] loss=0.973937 avg=0.843927 VRAM=39.5GiB | 65.1% done | ETA(epoch): 1086s + [Epoch 195][20/66] loss=0.883725 avg=0.850294 VRAM=39.2GiB | 65.1% done | ETA(epoch): 892s + [Epoch 195][30/66] loss=0.842179 avg=0.856150 VRAM=39.5GiB | 65.2% done | ETA(epoch): 698s + [Epoch 195][40/66] loss=0.976325 avg=0.861698 VRAM=39.2GiB | 65.2% done | ETA(epoch): 504s + [Epoch 195][50/66] loss=0.863721 avg=0.864932 VRAM=39.5GiB | 65.3% done | ETA(epoch): 310s + [Epoch 195][60/66] loss=0.914245 avg=0.861556 VRAM=39.2GiB | 65.3% done | ETA(epoch): 116s + Train loss: 0.859495 (1279.4s) ETA: 2429min + Val loss: 0.863251 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9505 t_0.4-0.6=0.8648 t_0.6-0.8=0.7380 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 196/299 (65% done) --- + [Epoch 196][10/66] loss=0.807216 avg=0.870051 VRAM=39.5GiB | 65.4% done | ETA(epoch): 1083s + [Epoch 196][20/66] loss=0.828456 avg=0.878769 VRAM=39.2GiB | 65.4% done | ETA(epoch): 891s + [Epoch 196][30/66] loss=0.904994 avg=0.873323 VRAM=39.5GiB | 65.5% done | ETA(epoch): 698s + [Epoch 196][40/66] loss=0.933623 avg=0.871858 VRAM=39.2GiB | 65.5% done | ETA(epoch): 505s + [Epoch 196][50/66] loss=0.985158 avg=0.863060 VRAM=39.5GiB | 65.6% done | ETA(epoch): 311s + [Epoch 196][60/66] loss=0.908391 avg=0.858657 VRAM=39.2GiB | 65.6% done | ETA(epoch): 116s + Train loss: 0.856673 (1281.2s) ETA: 2405min + Val loss: 0.823051 [t_0.0-0.2=1.0621 t_0.2-0.4=0.9745 t_0.4-0.6=0.8358 t_0.6-0.8=0.7151 t_0.8-1.0=0.6515] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 197/299 (66% done) --- + [Epoch 197][10/66] loss=0.732905 avg=0.880785 VRAM=39.5GiB | 65.7% done | ETA(epoch): 1086s + [Epoch 197][20/66] loss=0.866336 avg=0.860879 VRAM=39.2GiB | 65.8% done | ETA(epoch): 891s + [Epoch 197][30/66] loss=0.813637 avg=0.864446 VRAM=39.5GiB | 65.8% done | ETA(epoch): 697s + [Epoch 197][40/66] loss=0.731156 avg=0.853166 VRAM=39.2GiB | 65.9% done | ETA(epoch): 504s + [Epoch 197][50/66] loss=0.811525 avg=0.846472 VRAM=39.5GiB | 65.9% done | ETA(epoch): 310s + [Epoch 197][60/66] loss=0.918645 avg=0.847177 VRAM=39.2GiB | 66.0% done | ETA(epoch): 116s + Train loss: 0.849594 (1278.2s) ETA: 2382min + Val loss: 0.869997 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0050 t_0.4-0.6=0.8733 t_0.6-0.8=0.7223 t_0.8-1.0=0.6791] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 198/299 (66% done) --- + [Epoch 198][10/66] loss=0.685542 avg=0.823176 VRAM=39.5GiB | 66.1% done | ETA(epoch): 1083s + [Epoch 198][20/66] loss=0.805283 avg=0.827492 VRAM=39.2GiB | 66.1% done | ETA(epoch): 890s + [Epoch 198][30/66] loss=0.752974 avg=0.838621 VRAM=39.5GiB | 66.2% done | ETA(epoch): 696s + [Epoch 198][40/66] loss=0.934060 avg=0.840384 VRAM=39.2GiB | 66.2% done | ETA(epoch): 503s + [Epoch 198][50/66] loss=0.834522 avg=0.849561 VRAM=39.5GiB | 66.3% done | ETA(epoch): 309s + [Epoch 198][60/66] loss=0.871690 avg=0.856399 VRAM=39.2GiB | 66.3% done | ETA(epoch): 116s + Train loss: 0.856458 (1275.9s) ETA: 2358min + Val loss: 0.869541 [t_0.0-0.2=1.0753 t_0.2-0.4=0.9691 t_0.4-0.6=0.8570 t_0.6-0.8=0.7214 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 199/299 (66% done) --- + [Epoch 199][10/66] loss=0.847508 avg=0.873715 VRAM=39.5GiB | 66.4% done | ETA(epoch): 1101s + [Epoch 199][20/66] loss=0.983505 avg=0.862884 VRAM=39.2GiB | 66.4% done | ETA(epoch): 898s + [Epoch 199][30/66] loss=0.886524 avg=0.870449 VRAM=39.5GiB | 66.5% done | ETA(epoch): 699s + [Epoch 199][40/66] loss=0.833994 avg=0.864916 VRAM=39.2GiB | 66.5% done | ETA(epoch): 504s + [Epoch 199][50/66] loss=0.889321 avg=0.866045 VRAM=39.5GiB | 66.6% done | ETA(epoch): 310s + [Epoch 199][60/66] loss=0.844653 avg=0.859615 VRAM=39.2GiB | 66.6% done | ETA(epoch): 116s + Train loss: 0.860620 (1277.9s) ETA: 2335min + Val loss: 0.854180 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9868 t_0.4-0.6=0.8663 t_0.6-0.8=0.7275 t_0.8-1.0=0.6468] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 200/299 (67% done) --- + [Epoch 200][10/66] loss=0.903232 avg=0.869608 VRAM=39.5GiB | 66.7% done | ETA(epoch): 1082s + [Epoch 200][20/66] loss=0.792616 avg=0.878821 VRAM=39.2GiB | 66.8% done | ETA(epoch): 892s + [Epoch 200][30/66] loss=0.704111 avg=0.864506 VRAM=39.5GiB | 66.8% done | ETA(epoch): 698s + [Epoch 200][40/66] loss=0.868828 avg=0.859803 VRAM=39.2GiB | 66.9% done | ETA(epoch): 503s + [Epoch 200][50/66] loss=0.704667 avg=0.849503 VRAM=39.5GiB | 66.9% done | ETA(epoch): 310s + [Epoch 200][60/66] loss=0.920987 avg=0.848911 VRAM=39.2GiB | 67.0% done | ETA(epoch): 116s + Train loss: 0.848865 (1278.4s) ETA: 2311min + Val loss: 0.883305 [t_0.0-0.2=1.0847 t_0.2-0.4=0.9779 t_0.4-0.6=0.8437 t_0.6-0.8=0.7050 t_0.8-1.0=0.6682] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 201/299 (67% done) --- + [Epoch 201][10/66] loss=0.965628 avg=0.874996 VRAM=39.5GiB | 67.1% done | ETA(epoch): 1082s + [Epoch 201][20/66] loss=0.939759 avg=0.867307 VRAM=39.2GiB | 67.1% done | ETA(epoch): 890s + [Epoch 201][30/66] loss=0.823105 avg=0.876342 VRAM=39.5GiB | 67.2% done | ETA(epoch): 697s + [Epoch 201][40/66] loss=0.905728 avg=0.860523 VRAM=39.2GiB | 67.2% done | ETA(epoch): 503s + [Epoch 201][50/66] loss=0.874198 avg=0.861108 VRAM=39.5GiB | 67.3% done | ETA(epoch): 310s + [Epoch 201][60/66] loss=0.862290 avg=0.866999 VRAM=39.2GiB | 67.3% done | ETA(epoch): 116s + Train loss: 0.866684 (1277.3s) ETA: 2287min + Val loss: 0.842206 [t_0.0-0.2=1.0532 t_0.2-0.4=0.9959 t_0.4-0.6=0.8654 t_0.6-0.8=0.7172 t_0.8-1.0=0.6638] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 202/299 (67% done) --- + [Epoch 202][10/66] loss=0.932830 avg=0.834070 VRAM=39.5GiB | 67.4% done | ETA(epoch): 1084s + [Epoch 202][20/66] loss=0.713450 avg=0.820953 VRAM=39.2GiB | 67.4% done | ETA(epoch): 890s + [Epoch 202][30/66] loss=0.847080 avg=0.823678 VRAM=39.5GiB | 67.5% done | ETA(epoch): 696s + [Epoch 202][40/66] loss=0.922192 avg=0.825052 VRAM=39.2GiB | 67.5% done | ETA(epoch): 503s + [Epoch 202][50/66] loss=0.810303 avg=0.833926 VRAM=39.5GiB | 67.6% done | ETA(epoch): 309s + [Epoch 202][60/66] loss=0.946051 avg=0.842945 VRAM=39.2GiB | 67.6% done | ETA(epoch): 116s + Train loss: 0.838029 (1276.2s) ETA: 2264min + Val loss: 0.842274 [t_0.0-0.2=1.0916 t_0.2-0.4=0.9820 t_0.4-0.6=0.8453 t_0.6-0.8=0.6869 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 203/299 (68% done) --- + [Epoch 203][10/66] loss=0.759022 avg=0.836577 VRAM=39.5GiB | 67.7% done | ETA(epoch): 1081s + [Epoch 203][20/66] loss=0.957394 avg=0.859892 VRAM=39.2GiB | 67.8% done | ETA(epoch): 888s + [Epoch 203][30/66] loss=0.905787 avg=0.854809 VRAM=39.5GiB | 67.8% done | ETA(epoch): 695s + [Epoch 203][40/66] loss=0.863171 avg=0.861973 VRAM=39.2GiB | 67.9% done | ETA(epoch): 503s + [Epoch 203][50/66] loss=0.886403 avg=0.859975 VRAM=39.5GiB | 67.9% done | ETA(epoch): 310s + [Epoch 203][60/66] loss=0.852154 avg=0.854537 VRAM=39.2GiB | 68.0% done | ETA(epoch): 116s + Train loss: 0.853444 (1276.7s) ETA: 2240min + Val loss: 0.861892 [t_0.0-0.2=1.0764 t_0.2-0.4=0.9551 t_0.4-0.6=0.8664 t_0.6-0.8=0.7300 t_0.8-1.0=0.6629] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 204/299 (68% done) --- + [Epoch 204][10/66] loss=0.856734 avg=0.870215 VRAM=39.5GiB | 68.1% done | ETA(epoch): 1082s + [Epoch 204][20/66] loss=0.832200 avg=0.856014 VRAM=39.2GiB | 68.1% done | ETA(epoch): 889s + [Epoch 204][30/66] loss=0.910172 avg=0.865196 VRAM=39.5GiB | 68.2% done | ETA(epoch): 696s + [Epoch 204][40/66] loss=0.944411 avg=0.869658 VRAM=39.2GiB | 68.2% done | ETA(epoch): 503s + [Epoch 204][50/66] loss=0.996417 avg=0.870967 VRAM=39.5GiB | 68.3% done | ETA(epoch): 309s + [Epoch 204][60/66] loss=0.673122 avg=0.861747 VRAM=39.2GiB | 68.3% done | ETA(epoch): 116s + Train loss: 0.861500 (1276.0s) ETA: 2217min + Val loss: 0.858947 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0142 t_0.4-0.6=0.8216 t_0.6-0.8=0.7209 t_0.8-1.0=0.6461] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 205/299 (68% done) --- + [Epoch 205][10/66] loss=0.956294 avg=0.875530 VRAM=39.5GiB | 68.4% done | ETA(epoch): 1082s + [Epoch 205][20/66] loss=1.009210 avg=0.874619 VRAM=39.2GiB | 68.4% done | ETA(epoch): 895s + [Epoch 205][30/66] loss=0.851101 avg=0.871926 VRAM=39.5GiB | 68.5% done | ETA(epoch): 700s + [Epoch 205][40/66] loss=0.869139 avg=0.873153 VRAM=39.2GiB | 68.5% done | ETA(epoch): 505s + [Epoch 205][50/66] loss=0.909375 avg=0.860872 VRAM=39.5GiB | 68.6% done | ETA(epoch): 310s + [Epoch 205][60/66] loss=0.895709 avg=0.859245 VRAM=39.2GiB | 68.6% done | ETA(epoch): 116s + Train loss: 0.861080 (1279.1s) ETA: 2193min + Val loss: 0.845704 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9716 t_0.4-0.6=0.8806 t_0.6-0.8=0.7134 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 206/299 (69% done) --- + [Epoch 206][10/66] loss=0.968451 avg=0.903091 VRAM=39.5GiB | 68.7% done | ETA(epoch): 1097s + [Epoch 206][20/66] loss=0.865225 avg=0.885203 VRAM=39.2GiB | 68.8% done | ETA(epoch): 894s + [Epoch 206][30/66] loss=0.901274 avg=0.871786 VRAM=39.5GiB | 68.8% done | ETA(epoch): 699s + [Epoch 206][40/66] loss=0.810735 avg=0.857728 VRAM=39.2GiB | 68.9% done | ETA(epoch): 504s + [Epoch 206][50/66] loss=0.859330 avg=0.867993 VRAM=39.5GiB | 68.9% done | ETA(epoch): 310s + [Epoch 206][60/66] loss=0.806676 avg=0.868482 VRAM=39.2GiB | 69.0% done | ETA(epoch): 116s + Train loss: 0.870525 (1279.9s) ETA: 2170min + Val loss: 0.865513 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9972 t_0.4-0.6=0.8066 t_0.6-0.8=0.7253 t_0.8-1.0=0.6587] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 207/299 (69% done) --- + [Epoch 207][10/66] loss=0.724338 avg=0.826901 VRAM=39.5GiB | 69.1% done | ETA(epoch): 1081s + [Epoch 207][20/66] loss=0.825250 avg=0.844527 VRAM=39.2GiB | 69.1% done | ETA(epoch): 889s + [Epoch 207][30/66] loss=0.779752 avg=0.855186 VRAM=39.5GiB | 69.2% done | ETA(epoch): 696s + [Epoch 207][40/66] loss=0.850766 avg=0.854699 VRAM=39.2GiB | 69.2% done | ETA(epoch): 503s + [Epoch 207][50/66] loss=0.769860 avg=0.851263 VRAM=39.5GiB | 69.3% done | ETA(epoch): 310s + [Epoch 207][60/66] loss=0.903330 avg=0.848714 VRAM=39.2GiB | 69.3% done | ETA(epoch): 116s + Train loss: 0.849455 (1279.2s) ETA: 2146min + Val loss: 0.832289 [t_0.0-0.2=1.0613 t_0.2-0.4=0.9862 t_0.4-0.6=0.8187 t_0.6-0.8=0.7344 t_0.8-1.0=0.6482] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 208/299 (69% done) --- + [Epoch 208][10/66] loss=0.787189 avg=0.838262 VRAM=39.5GiB | 69.4% done | ETA(epoch): 1083s + [Epoch 208][20/66] loss=0.772324 avg=0.834155 VRAM=39.2GiB | 69.4% done | ETA(epoch): 891s + [Epoch 208][30/66] loss=0.861714 avg=0.840383 VRAM=39.5GiB | 69.5% done | ETA(epoch): 697s + [Epoch 208][40/66] loss=0.875539 avg=0.841400 VRAM=39.2GiB | 69.5% done | ETA(epoch): 503s + [Epoch 208][50/66] loss=0.784312 avg=0.847096 VRAM=39.5GiB | 69.6% done | ETA(epoch): 310s + [Epoch 208][60/66] loss=0.777958 avg=0.840115 VRAM=39.2GiB | 69.6% done | ETA(epoch): 116s + Train loss: 0.838947 (1277.3s) ETA: 2123min + Val loss: 0.826408 [t_0.0-0.2=1.0793 t_0.2-0.4=0.9828 t_0.4-0.6=0.8140 t_0.6-0.8=0.7086 t_0.8-1.0=0.6569] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 20.0/188.5 GiB (10.6%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 209/299 (70% done) --- + [Epoch 209][10/66] loss=0.917719 avg=0.868684 VRAM=39.5GiB | 69.7% done | ETA(epoch): 1084s + [Epoch 209][20/66] loss=0.844451 avg=0.854390 VRAM=39.2GiB | 69.8% done | ETA(epoch): 890s + [Epoch 209][30/66] loss=0.937267 avg=0.851408 VRAM=39.5GiB | 69.8% done | ETA(epoch): 697s + [Epoch 209][40/66] loss=0.885833 avg=0.849187 VRAM=39.2GiB | 69.9% done | ETA(epoch): 503s + [Epoch 209][50/66] loss=0.964025 avg=0.848062 VRAM=39.5GiB | 69.9% done | ETA(epoch): 311s + [Epoch 209][60/66] loss=0.946173 avg=0.856008 VRAM=39.2GiB | 70.0% done | ETA(epoch): 117s + [MilestoneVis] train_0 step 13859 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 13859 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 13859 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 13859 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 13859 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 13859 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 13859 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 13859 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 13859 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 13859 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + Train loss: 0.860893 (2313.8s) ETA: 2107min + Val loss: 0.842536 [t_0.0-0.2=1.0845 t_0.2-0.4=0.9963 t_0.4-0.6=0.8444 t_0.6-0.8=0.7104 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 210/299 (70% done) --- + [Epoch 210][10/66] loss=0.869734 avg=0.847135 VRAM=39.5GiB | 70.1% done | ETA(epoch): 1085s + [Epoch 210][20/66] loss=0.864142 avg=0.853584 VRAM=39.2GiB | 70.1% done | ETA(epoch): 891s + [Epoch 210][30/66] loss=0.720916 avg=0.850710 VRAM=39.5GiB | 70.2% done | ETA(epoch): 698s + [Epoch 210][40/66] loss=0.741375 avg=0.859190 VRAM=39.2GiB | 70.2% done | ETA(epoch): 504s + [Epoch 210][50/66] loss=0.829579 avg=0.853627 VRAM=39.5GiB | 70.3% done | ETA(epoch): 310s + [Epoch 210][60/66] loss=0.767255 avg=0.845728 VRAM=39.2GiB | 70.3% done | ETA(epoch): 116s + Train loss: 0.845002 (1280.0s) ETA: 2083min + Val loss: 0.864105 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9887 t_0.4-0.6=0.8521 t_0.6-0.8=0.7195 t_0.8-1.0=0.6479] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 211/299 (70% done) --- + [Epoch 211][10/66] loss=0.863808 avg=0.862833 VRAM=39.5GiB | 70.4% done | ETA(epoch): 1084s + [Epoch 211][20/66] loss=0.785054 avg=0.853057 VRAM=39.2GiB | 70.4% done | ETA(epoch): 890s + [Epoch 211][30/66] loss=0.927997 avg=0.860471 VRAM=39.5GiB | 70.5% done | ETA(epoch): 697s + [Epoch 211][40/66] loss=0.716332 avg=0.857551 VRAM=39.2GiB | 70.5% done | ETA(epoch): 503s + [Epoch 211][50/66] loss=0.891134 avg=0.855940 VRAM=39.5GiB | 70.6% done | ETA(epoch): 310s + [Epoch 211][60/66] loss=0.693832 avg=0.852313 VRAM=39.2GiB | 70.6% done | ETA(epoch): 116s + Train loss: 0.846927 (1279.3s) ETA: 2059min + Val loss: 0.828613 [t_0.0-0.2=1.0514 t_0.2-0.4=0.9726 t_0.4-0.6=0.8489 t_0.6-0.8=0.7089 t_0.8-1.0=0.6461] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 212/299 (71% done) --- + [Epoch 212][10/66] loss=0.852983 avg=0.833974 VRAM=39.5GiB | 70.7% done | ETA(epoch): 1085s + [Epoch 212][20/66] loss=1.060655 avg=0.852247 VRAM=39.2GiB | 70.8% done | ETA(epoch): 891s + [Epoch 212][30/66] loss=0.810860 avg=0.853182 VRAM=39.5GiB | 70.8% done | ETA(epoch): 699s + [Epoch 212][40/66] loss=0.821468 avg=0.854384 VRAM=39.2GiB | 70.9% done | ETA(epoch): 505s + [Epoch 212][50/66] loss=0.853087 avg=0.854072 VRAM=39.5GiB | 70.9% done | ETA(epoch): 310s + [Epoch 212][60/66] loss=0.825074 avg=0.856977 VRAM=39.2GiB | 71.0% done | ETA(epoch): 116s + Train loss: 0.858706 (1279.9s) ETA: 2036min + Val loss: 0.845496 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0030 t_0.4-0.6=0.8330 t_0.6-0.8=0.7208 t_0.8-1.0=0.6778] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 213/299 (71% done) --- + [Epoch 213][10/66] loss=0.892082 avg=0.922130 VRAM=39.5GiB | 71.1% done | ETA(epoch): 1084s + [Epoch 213][20/66] loss=0.833584 avg=0.886466 VRAM=39.2GiB | 71.1% done | ETA(epoch): 896s + [Epoch 213][30/66] loss=0.806421 avg=0.868386 VRAM=39.5GiB | 71.2% done | ETA(epoch): 701s + [Epoch 213][40/66] loss=0.819903 avg=0.864056 VRAM=39.2GiB | 71.2% done | ETA(epoch): 506s + [Epoch 213][50/66] loss=0.841295 avg=0.862065 VRAM=39.5GiB | 71.3% done | ETA(epoch): 311s + [Epoch 213][60/66] loss=0.841023 avg=0.859081 VRAM=39.2GiB | 71.3% done | ETA(epoch): 117s + Train loss: 0.856166 (1282.2s) ETA: 2012min + Val loss: 0.834490 [t_0.0-0.2=1.0559 t_0.2-0.4=1.0327 t_0.4-0.6=0.8551 t_0.6-0.8=0.7083 t_0.8-1.0=0.6544] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 214/299 (71% done) --- + [Epoch 214][10/66] loss=0.864257 avg=0.831050 VRAM=39.5GiB | 71.4% done | ETA(epoch): 1095s + [Epoch 214][20/66] loss=0.884096 avg=0.851531 VRAM=39.2GiB | 71.4% done | ETA(epoch): 896s + [Epoch 214][30/66] loss=0.814693 avg=0.839293 VRAM=39.5GiB | 71.5% done | ETA(epoch): 700s + [Epoch 214][40/66] loss=0.987028 avg=0.845118 VRAM=39.2GiB | 71.5% done | ETA(epoch): 506s + [Epoch 214][50/66] loss=0.795343 avg=0.848013 VRAM=39.5GiB | 71.6% done | ETA(epoch): 311s + [Epoch 214][60/66] loss=0.918602 avg=0.852660 VRAM=39.2GiB | 71.6% done | ETA(epoch): 117s + Train loss: 0.852730 (1282.0s) ETA: 1989min + Val loss: 0.858969 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9677 t_0.4-0.6=0.8629 t_0.6-0.8=0.7349 t_0.8-1.0=0.6578] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 215/299 (72% done) --- + [Epoch 215][10/66] loss=0.863853 avg=0.847578 VRAM=39.5GiB | 71.7% done | ETA(epoch): 1085s + [Epoch 215][20/66] loss=0.764150 avg=0.846150 VRAM=39.2GiB | 71.8% done | ETA(epoch): 892s + [Epoch 215][30/66] loss=0.799435 avg=0.852137 VRAM=39.5GiB | 71.8% done | ETA(epoch): 698s + [Epoch 215][40/66] loss=0.837897 avg=0.848785 VRAM=39.2GiB | 71.9% done | ETA(epoch): 504s + [Epoch 215][50/66] loss=0.872069 avg=0.851517 VRAM=39.5GiB | 71.9% done | ETA(epoch): 310s + [Epoch 215][60/66] loss=0.864786 avg=0.848197 VRAM=39.2GiB | 72.0% done | ETA(epoch): 116s + Train loss: 0.844701 (1283.8s) ETA: 1965min + Val loss: 0.853081 [t_0.0-0.2=1.1168 t_0.2-0.4=0.9926 t_0.4-0.6=0.8513 t_0.6-0.8=0.7151 t_0.8-1.0=0.6663] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 216/299 (72% done) --- + [Epoch 216][10/66] loss=0.875706 avg=0.850896 VRAM=39.5GiB | 72.1% done | ETA(epoch): 1083s + [Epoch 216][20/66] loss=0.804930 avg=0.848100 VRAM=39.2GiB | 72.1% done | ETA(epoch): 891s + [Epoch 216][30/66] loss=0.864209 avg=0.857562 VRAM=39.5GiB | 72.2% done | ETA(epoch): 697s + [Epoch 216][40/66] loss=0.890773 avg=0.862623 VRAM=39.2GiB | 72.2% done | ETA(epoch): 503s + [Epoch 216][50/66] loss=0.911622 avg=0.867313 VRAM=39.5GiB | 72.3% done | ETA(epoch): 309s + [Epoch 216][60/66] loss=0.812213 avg=0.868051 VRAM=39.2GiB | 72.3% done | ETA(epoch): 116s + Train loss: 0.865674 (1275.0s) ETA: 1941min + Val loss: 0.870027 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9830 t_0.4-0.6=0.8562 t_0.6-0.8=0.7079 t_0.8-1.0=0.6560] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 217/299 (72% done) --- + [Epoch 217][10/66] loss=0.886144 avg=0.825575 VRAM=39.5GiB | 72.4% done | ETA(epoch): 1081s + [Epoch 217][20/66] loss=0.920118 avg=0.837542 VRAM=39.2GiB | 72.4% done | ETA(epoch): 886s + [Epoch 217][30/66] loss=0.713349 avg=0.850834 VRAM=39.5GiB | 72.5% done | ETA(epoch): 694s + [Epoch 217][40/66] loss=0.855782 avg=0.846722 VRAM=39.2GiB | 72.5% done | ETA(epoch): 501s + [Epoch 217][50/66] loss=0.860780 avg=0.851864 VRAM=39.5GiB | 72.6% done | ETA(epoch): 309s + [Epoch 217][60/66] loss=0.909450 avg=0.854497 VRAM=39.2GiB | 72.6% done | ETA(epoch): 116s + Train loss: 0.855419 (1272.7s) ETA: 1918min + Val loss: 0.862234 [t_0.0-0.2=1.0642 t_0.2-0.4=0.9816 t_0.4-0.6=0.8693 t_0.6-0.8=0.7415 t_0.8-1.0=0.6394] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 218/299 (73% done) --- + [Epoch 218][10/66] loss=0.848709 avg=0.895708 VRAM=39.5GiB | 72.7% done | ETA(epoch): 1080s + [Epoch 218][20/66] loss=0.847892 avg=0.870299 VRAM=39.2GiB | 72.8% done | ETA(epoch): 888s + [Epoch 218][30/66] loss=0.806536 avg=0.853646 VRAM=39.5GiB | 72.8% done | ETA(epoch): 695s + [Epoch 218][40/66] loss=0.769769 avg=0.846711 VRAM=39.2GiB | 72.9% done | ETA(epoch): 502s + [Epoch 218][50/66] loss=0.922476 avg=0.842242 VRAM=39.5GiB | 72.9% done | ETA(epoch): 309s + [Epoch 218][60/66] loss=0.869285 avg=0.848052 VRAM=39.2GiB | 73.0% done | ETA(epoch): 116s + Train loss: 0.846748 (1274.3s) ETA: 1894min + Val loss: 0.835833 [t_0.0-0.2=1.0857 t_0.2-0.4=0.9908 t_0.4-0.6=0.7974 t_0.6-0.8=0.7156 t_0.8-1.0=0.6591] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 219/299 (73% done) --- + [Epoch 219][10/66] loss=0.827884 avg=0.855040 VRAM=39.5GiB | 73.1% done | ETA(epoch): 1079s + [Epoch 219][20/66] loss=0.737465 avg=0.823496 VRAM=39.2GiB | 73.1% done | ETA(epoch): 888s + [Epoch 219][30/66] loss=0.792870 avg=0.841305 VRAM=39.5GiB | 73.2% done | ETA(epoch): 697s + [Epoch 219][40/66] loss=1.026051 avg=0.850399 VRAM=39.2GiB | 73.2% done | ETA(epoch): 503s + [Epoch 219][50/66] loss=0.881046 avg=0.851641 VRAM=39.5GiB | 73.3% done | ETA(epoch): 309s + [Epoch 219][60/66] loss=0.859167 avg=0.857518 VRAM=39.2GiB | 73.3% done | ETA(epoch): 116s + Train loss: 0.858820 (1276.0s) ETA: 1871min + Val loss: 0.853431 [t_0.0-0.2=1.0551 t_0.2-0.4=0.9709 t_0.4-0.6=0.8443 t_0.6-0.8=0.7096 t_0.8-1.0=0.6490] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 220/299 (73% done) --- + [Epoch 220][10/66] loss=0.782996 avg=0.836319 VRAM=39.5GiB | 73.4% done | ETA(epoch): 1079s + [Epoch 220][20/66] loss=0.863851 avg=0.847449 VRAM=39.2GiB | 73.4% done | ETA(epoch): 891s + [Epoch 220][30/66] loss=0.818546 avg=0.851528 VRAM=39.5GiB | 73.5% done | ETA(epoch): 696s + [Epoch 220][40/66] loss=0.876520 avg=0.850873 VRAM=39.2GiB | 73.5% done | ETA(epoch): 502s + [Epoch 220][50/66] loss=0.968734 avg=0.854998 VRAM=39.5GiB | 73.6% done | ETA(epoch): 309s + [Epoch 220][60/66] loss=0.813224 avg=0.853005 VRAM=39.2GiB | 73.6% done | ETA(epoch): 116s + Train loss: 0.850051 (1273.8s) ETA: 1847min + Val loss: 0.841740 [t_0.0-0.2=1.0605 t_0.2-0.4=1.0157 t_0.4-0.6=0.8251 t_0.6-0.8=0.7133 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 221/299 (74% done) --- + [Epoch 221][10/66] loss=0.835836 avg=0.861383 VRAM=39.5GiB | 73.7% done | ETA(epoch): 1078s + [Epoch 221][20/66] loss=0.758292 avg=0.865426 VRAM=39.2GiB | 73.8% done | ETA(epoch): 886s + [Epoch 221][30/66] loss=0.881543 avg=0.861972 VRAM=39.5GiB | 73.8% done | ETA(epoch): 693s + [Epoch 221][40/66] loss=0.862039 avg=0.854441 VRAM=39.2GiB | 73.9% done | ETA(epoch): 501s + [Epoch 221][50/66] loss=0.820564 avg=0.850053 VRAM=39.5GiB | 73.9% done | ETA(epoch): 308s + [Epoch 221][60/66] loss=0.838977 avg=0.853164 VRAM=39.2GiB | 74.0% done | ETA(epoch): 116s + Train loss: 0.848663 (1271.5s) ETA: 1823min + Val loss: 0.856184 [t_0.0-0.2=1.0752 t_0.2-0.4=0.9823 t_0.4-0.6=0.8369 t_0.6-0.8=0.7244 t_0.8-1.0=0.6680] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 222/299 (74% done) --- + [Epoch 222][10/66] loss=0.851335 avg=0.859796 VRAM=39.5GiB | 74.1% done | ETA(epoch): 1087s + [Epoch 222][20/66] loss=0.773987 avg=0.851269 VRAM=39.2GiB | 74.1% done | ETA(epoch): 889s + [Epoch 222][30/66] loss=0.847308 avg=0.856343 VRAM=39.5GiB | 74.2% done | ETA(epoch): 695s + [Epoch 222][40/66] loss=0.983880 avg=0.849834 VRAM=39.2GiB | 74.2% done | ETA(epoch): 502s + [Epoch 222][50/66] loss=0.780149 avg=0.848782 VRAM=39.5GiB | 74.3% done | ETA(epoch): 309s + [Epoch 222][60/66] loss=0.734389 avg=0.842575 VRAM=39.2GiB | 74.3% done | ETA(epoch): 116s + Train loss: 0.845464 (1276.4s) ETA: 1800min + Val loss: 0.882951 [t_0.0-0.2=1.0542 t_0.2-0.4=1.0152 t_0.4-0.6=0.8571 t_0.6-0.8=0.7055 t_0.8-1.0=0.6626] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 223/299 (74% done) --- + [Epoch 223][10/66] loss=0.816649 avg=0.818953 VRAM=39.5GiB | 74.4% done | ETA(epoch): 1079s + [Epoch 223][20/66] loss=0.888983 avg=0.835556 VRAM=39.2GiB | 74.4% done | ETA(epoch): 887s + [Epoch 223][30/66] loss=0.859664 avg=0.852593 VRAM=39.5GiB | 74.5% done | ETA(epoch): 694s + [Epoch 223][40/66] loss=0.953299 avg=0.847174 VRAM=39.2GiB | 74.5% done | ETA(epoch): 501s + [Epoch 223][50/66] loss=0.879306 avg=0.854129 VRAM=39.5GiB | 74.6% done | ETA(epoch): 308s + [Epoch 223][60/66] loss=0.781236 avg=0.847367 VRAM=39.2GiB | 74.6% done | ETA(epoch): 116s + Train loss: 0.851671 (1272.8s) ETA: 1776min + Val loss: 0.850706 [t_0.0-0.2=1.0852 t_0.2-0.4=0.9987 t_0.4-0.6=0.8127 t_0.6-0.8=0.7223 t_0.8-1.0=0.6614] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 224/299 (75% done) --- + [Epoch 224][10/66] loss=0.716457 avg=0.844708 VRAM=39.5GiB | 74.7% done | ETA(epoch): 1079s + [Epoch 224][20/66] loss=0.841949 avg=0.849106 VRAM=39.2GiB | 74.8% done | ETA(epoch): 886s + [Epoch 224][30/66] loss=0.929774 avg=0.850581 VRAM=39.5GiB | 74.8% done | ETA(epoch): 693s + [Epoch 224][40/66] loss=0.847706 avg=0.853692 VRAM=39.2GiB | 74.9% done | ETA(epoch): 501s + [Epoch 224][50/66] loss=0.809348 avg=0.858955 VRAM=39.5GiB | 74.9% done | ETA(epoch): 308s + [Epoch 224][60/66] loss=0.837513 avg=0.853837 VRAM=39.2GiB | 75.0% done | ETA(epoch): 116s + Train loss: 0.856708 (1272.3s) ETA: 1753min + Val loss: 0.845646 [t_0.0-0.2=1.0563 t_0.2-0.4=1.0064 t_0.4-0.6=0.8591 t_0.6-0.8=0.7039 t_0.8-1.0=0.6601] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 225/299 (75% done) --- + [Epoch 225][10/66] loss=0.840042 avg=0.879431 VRAM=39.5GiB | 75.1% done | ETA(epoch): 1081s + [Epoch 225][20/66] loss=0.958682 avg=0.870874 VRAM=39.2GiB | 75.1% done | ETA(epoch): 887s + [Epoch 225][30/66] loss=0.916305 avg=0.868151 VRAM=39.5GiB | 75.2% done | ETA(epoch): 694s + [Epoch 225][40/66] loss=0.923012 avg=0.877908 VRAM=39.2GiB | 75.2% done | ETA(epoch): 502s + [Epoch 225][50/66] loss=0.729540 avg=0.871628 VRAM=39.5GiB | 75.3% done | ETA(epoch): 309s + [Epoch 225][60/66] loss=0.936607 avg=0.874368 VRAM=39.2GiB | 75.3% done | ETA(epoch): 116s + Train loss: 0.872955 (1272.8s) ETA: 1729min + Val loss: 0.873647 [t_0.0-0.2=1.0665 t_0.2-0.4=1.0416 t_0.4-0.6=0.8346 t_0.6-0.8=0.7167 t_0.8-1.0=0.6522] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 226/299 (75% done) --- + [Epoch 226][10/66] loss=0.815075 avg=0.867276 VRAM=39.5GiB | 75.4% done | ETA(epoch): 1077s + [Epoch 226][20/66] loss=0.828218 avg=0.867467 VRAM=39.2GiB | 75.4% done | ETA(epoch): 886s + [Epoch 226][30/66] loss=0.737130 avg=0.861617 VRAM=39.5GiB | 75.5% done | ETA(epoch): 702s + [Epoch 226][40/66] loss=0.887109 avg=0.871811 VRAM=39.2GiB | 75.5% done | ETA(epoch): 505s + [Epoch 226][50/66] loss=0.851880 avg=0.875125 VRAM=39.5GiB | 75.6% done | ETA(epoch): 311s + [Epoch 226][60/66] loss=1.009094 avg=0.878974 VRAM=39.2GiB | 75.6% done | ETA(epoch): 116s + Train loss: 0.873225 (1278.4s) ETA: 1706min + Val loss: 0.850136 [t_0.0-0.2=1.0553 t_0.2-0.4=1.0083 t_0.4-0.6=0.8479 t_0.6-0.8=0.7289 t_0.8-1.0=0.6561] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 227/299 (76% done) --- + [Epoch 227][10/66] loss=0.993252 avg=0.843666 VRAM=39.5GiB | 75.7% done | ETA(epoch): 1078s + [Epoch 227][20/66] loss=0.877819 avg=0.845522 VRAM=39.2GiB | 75.8% done | ETA(epoch): 885s + [Epoch 227][30/66] loss=0.809538 avg=0.845702 VRAM=39.5GiB | 75.8% done | ETA(epoch): 693s + [Epoch 227][40/66] loss=0.806255 avg=0.839448 VRAM=39.2GiB | 75.9% done | ETA(epoch): 501s + [Epoch 227][50/66] loss=0.838483 avg=0.841680 VRAM=39.5GiB | 75.9% done | ETA(epoch): 308s + [Epoch 227][60/66] loss=0.890671 avg=0.848304 VRAM=39.2GiB | 76.0% done | ETA(epoch): 116s + Train loss: 0.849935 (1270.9s) ETA: 1682min + Val loss: 0.830520 [t_0.0-0.2=1.0602 t_0.2-0.4=0.9788 t_0.4-0.6=0.8834 t_0.6-0.8=0.6976 t_0.8-1.0=0.6633] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 228/299 (76% done) --- + [Epoch 228][10/66] loss=0.924953 avg=0.869359 VRAM=39.5GiB | 76.1% done | ETA(epoch): 1077s + [Epoch 228][20/66] loss=0.727478 avg=0.850087 VRAM=39.2GiB | 76.1% done | ETA(epoch): 889s + [Epoch 228][30/66] loss=0.776668 avg=0.851331 VRAM=39.5GiB | 76.2% done | ETA(epoch): 695s + [Epoch 228][40/66] loss=0.751664 avg=0.844757 VRAM=39.2GiB | 76.2% done | ETA(epoch): 502s + [Epoch 228][50/66] loss=0.921680 avg=0.842998 VRAM=39.5GiB | 76.3% done | ETA(epoch): 309s + [Epoch 228][60/66] loss=0.760058 avg=0.841202 VRAM=39.2GiB | 76.3% done | ETA(epoch): 116s + Train loss: 0.839255 (1272.1s) ETA: 1659min + Val loss: 0.863963 [t_0.0-0.2=1.0812 t_0.2-0.4=1.0248 t_0.4-0.6=0.8368 t_0.6-0.8=0.7223 t_0.8-1.0=0.6546] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 229/299 (76% done) --- + [Epoch 229][10/66] loss=0.738023 avg=0.848839 VRAM=39.5GiB | 76.4% done | ETA(epoch): 1076s + [Epoch 229][20/66] loss=0.830565 avg=0.847227 VRAM=39.2GiB | 76.4% done | ETA(epoch): 885s + [Epoch 229][30/66] loss=0.903151 avg=0.856898 VRAM=39.5GiB | 76.5% done | ETA(epoch): 693s + [Epoch 229][40/66] loss=0.803676 avg=0.853891 VRAM=39.2GiB | 76.5% done | ETA(epoch): 501s + [Epoch 229][50/66] loss=0.847994 avg=0.857586 VRAM=39.5GiB | 76.6% done | ETA(epoch): 308s + [Epoch 229][60/66] loss=0.784233 avg=0.858405 VRAM=39.2GiB | 76.6% done | ETA(epoch): 116s + Train loss: 0.855486 (1271.2s) ETA: 1635min + Val loss: 0.843292 [t_0.0-0.2=1.0627 t_0.2-0.4=1.0148 t_0.4-0.6=0.8429 t_0.6-0.8=0.7158 t_0.8-1.0=0.6513] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 230/299 (77% done) --- + [Epoch 230][10/66] loss=0.828293 avg=0.821324 VRAM=39.5GiB | 76.7% done | ETA(epoch): 1075s + [Epoch 230][20/66] loss=0.936386 avg=0.829925 VRAM=39.2GiB | 76.8% done | ETA(epoch): 884s + [Epoch 230][30/66] loss=0.765479 avg=0.837324 VRAM=39.5GiB | 76.8% done | ETA(epoch): 692s + [Epoch 230][40/66] loss=0.926227 avg=0.857603 VRAM=39.2GiB | 76.9% done | ETA(epoch): 500s + [Epoch 230][50/66] loss=0.948884 avg=0.852224 VRAM=39.5GiB | 76.9% done | ETA(epoch): 308s + [Epoch 230][60/66] loss=0.770136 avg=0.851842 VRAM=39.2GiB | 77.0% done | ETA(epoch): 116s + Train loss: 0.856009 (1273.0s) ETA: 1612min + Val loss: 0.839881 [t_0.0-0.2=1.0747 t_0.2-0.4=0.9765 t_0.4-0.6=0.8590 t_0.6-0.8=0.7191 t_0.8-1.0=0.6626] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 231/299 (77% done) --- + [Epoch 231][10/66] loss=0.947152 avg=0.847779 VRAM=39.5GiB | 77.1% done | ETA(epoch): 1078s + [Epoch 231][20/66] loss=0.890649 avg=0.871058 VRAM=39.2GiB | 77.1% done | ETA(epoch): 887s + [Epoch 231][30/66] loss=0.816745 avg=0.861569 VRAM=39.5GiB | 77.2% done | ETA(epoch): 694s + [Epoch 231][40/66] loss=0.786852 avg=0.861945 VRAM=39.2GiB | 77.2% done | ETA(epoch): 501s + [Epoch 231][50/66] loss=0.731586 avg=0.859452 VRAM=39.5GiB | 77.3% done | ETA(epoch): 309s + [Epoch 231][60/66] loss=0.915026 avg=0.859684 VRAM=39.2GiB | 77.3% done | ETA(epoch): 116s + Train loss: 0.863402 (1272.6s) ETA: 1588min + Val loss: 0.855723 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9966 t_0.4-0.6=0.8511 t_0.6-0.8=0.7037 t_0.8-1.0=0.6550] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 232/299 (77% done) --- + [Epoch 232][10/66] loss=0.856601 avg=0.885599 VRAM=39.5GiB | 77.4% done | ETA(epoch): 1076s + [Epoch 232][20/66] loss=0.830079 avg=0.862319 VRAM=39.2GiB | 77.4% done | ETA(epoch): 885s + [Epoch 232][30/66] loss=1.036203 avg=0.862683 VRAM=39.5GiB | 77.5% done | ETA(epoch): 693s + [Epoch 232][40/66] loss=0.993161 avg=0.869396 VRAM=39.2GiB | 77.5% done | ETA(epoch): 500s + [Epoch 232][50/66] loss=0.808444 avg=0.866516 VRAM=39.5GiB | 77.6% done | ETA(epoch): 308s + [Epoch 232][60/66] loss=0.937821 avg=0.861730 VRAM=39.2GiB | 77.6% done | ETA(epoch): 115s + Train loss: 0.857236 (1270.2s) ETA: 1564min + Val loss: 0.849562 [t_0.0-0.2=1.0712 t_0.2-0.4=0.9860 t_0.4-0.6=0.7983 t_0.6-0.8=0.7037 t_0.8-1.0=0.6614] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 233/299 (78% done) --- + [Epoch 233][10/66] loss=0.908471 avg=0.861317 VRAM=39.5GiB | 77.7% done | ETA(epoch): 1079s + [Epoch 233][20/66] loss=0.800512 avg=0.868071 VRAM=39.2GiB | 77.8% done | ETA(epoch): 886s + [Epoch 233][30/66] loss=0.851389 avg=0.877212 VRAM=39.5GiB | 77.8% done | ETA(epoch): 693s + [Epoch 233][40/66] loss=0.847370 avg=0.875254 VRAM=39.2GiB | 77.9% done | ETA(epoch): 502s + [Epoch 233][50/66] loss=0.829672 avg=0.874962 VRAM=39.5GiB | 77.9% done | ETA(epoch): 309s + [Epoch 233][60/66] loss=0.837947 avg=0.873054 VRAM=39.2GiB | 78.0% done | ETA(epoch): 116s + Train loss: 0.872657 (1272.9s) ETA: 1541min + Val loss: 0.867856 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0004 t_0.4-0.6=0.8076 t_0.6-0.8=0.7192 t_0.8-1.0=0.6538] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 234/299 (78% done) --- + [Epoch 234][10/66] loss=0.809526 avg=0.867634 VRAM=39.5GiB | 78.1% done | ETA(epoch): 1078s + [Epoch 234][20/66] loss=0.877382 avg=0.867712 VRAM=39.2GiB | 78.1% done | ETA(epoch): 885s + [Epoch 234][30/66] loss=0.923710 avg=0.860368 VRAM=39.5GiB | 78.2% done | ETA(epoch): 693s + [Epoch 234][40/66] loss=0.961232 avg=0.866756 VRAM=39.2GiB | 78.2% done | ETA(epoch): 501s + [Epoch 234][50/66] loss=0.901686 avg=0.862071 VRAM=39.5GiB | 78.3% done | ETA(epoch): 308s + [Epoch 234][60/66] loss=0.962167 avg=0.861623 VRAM=39.2GiB | 78.3% done | ETA(epoch): 116s + Train loss: 0.862291 (1271.6s) ETA: 1517min + Val loss: 0.844367 [t_0.0-0.2=1.0792 t_0.2-0.4=0.9818 t_0.4-0.6=0.8524 t_0.6-0.8=0.7059 t_0.8-1.0=0.6673] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 235/299 (78% done) --- + [Epoch 235][10/66] loss=0.972675 avg=0.900563 VRAM=39.5GiB | 78.4% done | ETA(epoch): 1077s + [Epoch 235][20/66] loss=0.890721 avg=0.875305 VRAM=39.2GiB | 78.4% done | ETA(epoch): 884s + [Epoch 235][30/66] loss=0.998487 avg=0.875987 VRAM=39.5GiB | 78.5% done | ETA(epoch): 693s + [Epoch 235][40/66] loss=0.935014 avg=0.874100 VRAM=39.2GiB | 78.5% done | ETA(epoch): 501s + [Epoch 235][50/66] loss=0.828826 avg=0.865825 VRAM=39.5GiB | 78.6% done | ETA(epoch): 308s + [Epoch 235][60/66] loss=0.785799 avg=0.858935 VRAM=39.2GiB | 78.6% done | ETA(epoch): 116s + Train loss: 0.860704 (1271.1s) ETA: 1494min + Val loss: 0.859985 [t_0.0-0.2=1.0834 t_0.2-0.4=0.9959 t_0.4-0.6=0.8454 t_0.6-0.8=0.7137 t_0.8-1.0=0.6355] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 236/299 (79% done) --- + [Epoch 236][10/66] loss=0.775130 avg=0.844431 VRAM=39.5GiB | 78.7% done | ETA(epoch): 1078s + [Epoch 236][20/66] loss=0.804586 avg=0.871801 VRAM=39.2GiB | 78.8% done | ETA(epoch): 886s + [Epoch 236][30/66] loss=0.884842 avg=0.870078 VRAM=39.5GiB | 78.8% done | ETA(epoch): 693s + [Epoch 236][40/66] loss=0.815416 avg=0.866854 VRAM=39.2GiB | 78.9% done | ETA(epoch): 500s + [Epoch 236][50/66] loss=1.036535 avg=0.859793 VRAM=39.5GiB | 78.9% done | ETA(epoch): 308s + [Epoch 236][60/66] loss=0.792866 avg=0.855140 VRAM=39.2GiB | 79.0% done | ETA(epoch): 116s + Train loss: 0.856513 (1270.6s) ETA: 1470min + Val loss: 0.886833 [t_0.0-0.2=1.0826 t_0.2-0.4=1.0009 t_0.4-0.6=0.8572 t_0.6-0.8=0.6870 t_0.8-1.0=0.6490] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 237/299 (79% done) --- + [Epoch 237][10/66] loss=0.977933 avg=0.892325 VRAM=39.5GiB | 79.1% done | ETA(epoch): 1078s + [Epoch 237][20/66] loss=0.875823 avg=0.875442 VRAM=39.2GiB | 79.1% done | ETA(epoch): 886s + [Epoch 237][30/66] loss=0.946186 avg=0.886694 VRAM=39.5GiB | 79.2% done | ETA(epoch): 693s + [Epoch 237][40/66] loss=0.788495 avg=0.885654 VRAM=39.2GiB | 79.2% done | ETA(epoch): 501s + [Epoch 237][50/66] loss=0.934727 avg=0.878195 VRAM=39.5GiB | 79.3% done | ETA(epoch): 308s + [Epoch 237][60/66] loss=0.753490 avg=0.870634 VRAM=39.2GiB | 79.3% done | ETA(epoch): 116s + Train loss: 0.869307 (1271.3s) ETA: 1447min + Val loss: 0.866473 [t_0.0-0.2=1.0756 t_0.2-0.4=0.9970 t_0.4-0.6=0.8133 t_0.6-0.8=0.7304 t_0.8-1.0=0.6687] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 238/299 (79% done) --- + [Epoch 238][10/66] loss=0.943640 avg=0.848529 VRAM=39.5GiB | 79.4% done | ETA(epoch): 1077s + [Epoch 238][20/66] loss=0.762922 avg=0.836330 VRAM=39.2GiB | 79.4% done | ETA(epoch): 886s + [Epoch 238][30/66] loss=0.933084 avg=0.843052 VRAM=39.5GiB | 79.5% done | ETA(epoch): 693s + [Epoch 238][40/66] loss=0.758604 avg=0.847664 VRAM=39.2GiB | 79.5% done | ETA(epoch): 501s + [Epoch 238][50/66] loss=0.860784 avg=0.850656 VRAM=39.5GiB | 79.6% done | ETA(epoch): 308s + [Epoch 238][60/66] loss=0.955873 avg=0.851629 VRAM=39.2GiB | 79.6% done | ETA(epoch): 116s + Train loss: 0.845791 (1273.4s) ETA: 1424min + Val loss: 0.846960 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0355 t_0.4-0.6=0.8772 t_0.6-0.8=0.6872 t_0.8-1.0=0.6554] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 239/299 (80% done) --- + [Epoch 239][10/66] loss=0.912125 avg=0.838448 VRAM=39.5GiB | 79.7% done | ETA(epoch): 1077s + [Epoch 239][20/66] loss=0.852308 avg=0.853212 VRAM=39.2GiB | 79.8% done | ETA(epoch): 885s + [Epoch 239][30/66] loss=0.761421 avg=0.859352 VRAM=39.5GiB | 79.8% done | ETA(epoch): 692s + [Epoch 239][40/66] loss=0.805682 avg=0.846177 VRAM=39.2GiB | 79.9% done | ETA(epoch): 500s + [Epoch 239][50/66] loss=0.821315 avg=0.851272 VRAM=39.5GiB | 79.9% done | ETA(epoch): 308s + [Epoch 239][60/66] loss=0.898896 avg=0.851469 VRAM=39.2GiB | 80.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 15839 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 15839 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 15839 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 15839 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 15839 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 15839 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 15839 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 15839 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 15839 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 15839 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + Train loss: 0.852515 (2307.1s) ETA: 1404min + Val loss: 0.863938 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9919 t_0.4-0.6=0.8461 t_0.6-0.8=0.7133 t_0.8-1.0=0.6704] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 240/299 (80% done) --- + [Epoch 240][10/66] loss=0.773764 avg=0.812767 VRAM=39.5GiB | 80.1% done | ETA(epoch): 1078s + [Epoch 240][20/66] loss=0.756155 avg=0.832399 VRAM=39.2GiB | 80.1% done | ETA(epoch): 886s + [Epoch 240][30/66] loss=0.742079 avg=0.848919 VRAM=39.5GiB | 80.2% done | ETA(epoch): 693s + [Epoch 240][40/66] loss=0.802675 avg=0.852369 VRAM=39.2GiB | 80.2% done | ETA(epoch): 500s + [Epoch 240][50/66] loss=0.877618 avg=0.858232 VRAM=39.5GiB | 80.3% done | ETA(epoch): 309s + [Epoch 240][60/66] loss=0.732692 avg=0.857405 VRAM=39.2GiB | 80.3% done | ETA(epoch): 116s + Train loss: 0.854743 (1273.2s) ETA: 1381min + Val loss: 0.857660 [t_0.0-0.2=1.0521 t_0.2-0.4=0.9753 t_0.4-0.6=0.8340 t_0.6-0.8=0.7119 t_0.8-1.0=0.6686] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 241/299 (80% done) --- + [Epoch 241][10/66] loss=1.026201 avg=0.891843 VRAM=39.5GiB | 80.4% done | ETA(epoch): 1079s + [Epoch 241][20/66] loss=0.796088 avg=0.879004 VRAM=39.2GiB | 80.4% done | ETA(epoch): 887s + [Epoch 241][30/66] loss=1.032958 avg=0.866757 VRAM=39.5GiB | 80.5% done | ETA(epoch): 694s + [Epoch 241][40/66] loss=0.914052 avg=0.860081 VRAM=39.2GiB | 80.5% done | ETA(epoch): 503s + [Epoch 241][50/66] loss=0.726689 avg=0.855763 VRAM=39.5GiB | 80.6% done | ETA(epoch): 309s + [Epoch 241][60/66] loss=0.817720 avg=0.853717 VRAM=39.2GiB | 80.6% done | ETA(epoch): 116s + Train loss: 0.852313 (1273.4s) ETA: 1357min + Val loss: 0.854300 [t_0.0-0.2=1.0587 t_0.2-0.4=1.0120 t_0.4-0.6=0.8246 t_0.6-0.8=0.7142 t_0.8-1.0=0.6510] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 242/299 (81% done) --- + [Epoch 242][10/66] loss=0.853196 avg=0.834391 VRAM=39.5GiB | 80.7% done | ETA(epoch): 1078s + [Epoch 242][20/66] loss=0.766457 avg=0.832777 VRAM=39.2GiB | 80.8% done | ETA(epoch): 886s + [Epoch 242][30/66] loss=0.894208 avg=0.849650 VRAM=39.5GiB | 80.8% done | ETA(epoch): 695s + [Epoch 242][40/66] loss=0.802866 avg=0.847380 VRAM=39.2GiB | 80.9% done | ETA(epoch): 502s + [Epoch 242][50/66] loss=0.741502 avg=0.846960 VRAM=39.5GiB | 80.9% done | ETA(epoch): 309s + [Epoch 242][60/66] loss=0.728460 avg=0.852567 VRAM=39.2GiB | 81.0% done | ETA(epoch): 116s + Train loss: 0.852217 (1272.7s) ETA: 1334min + Val loss: 0.867701 [t_0.0-0.2=1.0594 t_0.2-0.4=1.0010 t_0.4-0.6=0.8550 t_0.6-0.8=0.7313 t_0.8-1.0=0.6692] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 243/299 (81% done) --- + [Epoch 243][10/66] loss=0.786319 avg=0.843213 VRAM=39.5GiB | 81.1% done | ETA(epoch): 1078s + [Epoch 243][20/66] loss=0.918152 avg=0.851770 VRAM=39.2GiB | 81.1% done | ETA(epoch): 886s + [Epoch 243][30/66] loss=0.963165 avg=0.850915 VRAM=39.5GiB | 81.2% done | ETA(epoch): 693s + [Epoch 243][40/66] loss=0.854499 avg=0.849094 VRAM=39.2GiB | 81.2% done | ETA(epoch): 501s + [Epoch 243][50/66] loss=0.929572 avg=0.854427 VRAM=39.5GiB | 81.3% done | ETA(epoch): 308s + [Epoch 243][60/66] loss=0.884697 avg=0.861778 VRAM=39.2GiB | 81.3% done | ETA(epoch): 116s + Train loss: 0.862065 (1271.0s) ETA: 1310min + Val loss: 0.831395 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0134 t_0.4-0.6=0.8236 t_0.6-0.8=0.7118 t_0.8-1.0=0.6590] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 244/299 (81% done) --- + [Epoch 244][10/66] loss=0.772058 avg=0.841964 VRAM=39.5GiB | 81.4% done | ETA(epoch): 1078s + [Epoch 244][20/66] loss=0.923225 avg=0.842357 VRAM=39.2GiB | 81.4% done | ETA(epoch): 890s + [Epoch 244][30/66] loss=0.732284 avg=0.840330 VRAM=39.5GiB | 81.5% done | ETA(epoch): 695s + [Epoch 244][40/66] loss=0.820384 avg=0.849003 VRAM=39.2GiB | 81.5% done | ETA(epoch): 502s + [Epoch 244][50/66] loss=0.891850 avg=0.850128 VRAM=39.5GiB | 81.6% done | ETA(epoch): 309s + [Epoch 244][60/66] loss=0.884360 avg=0.841041 VRAM=39.2GiB | 81.6% done | ETA(epoch): 116s + Train loss: 0.841068 (1272.3s) ETA: 1287min + Val loss: 0.851914 [t_0.0-0.2=1.0602 t_0.2-0.4=1.0072 t_0.4-0.6=0.8341 t_0.6-0.8=0.7283 t_0.8-1.0=0.6484] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 245/299 (82% done) --- + [Epoch 245][10/66] loss=0.888119 avg=0.869851 VRAM=39.5GiB | 81.7% done | ETA(epoch): 1083s + [Epoch 245][20/66] loss=0.817106 avg=0.851155 VRAM=39.2GiB | 81.8% done | ETA(epoch): 888s + [Epoch 245][30/66] loss=0.830420 avg=0.864501 VRAM=39.5GiB | 81.8% done | ETA(epoch): 695s + [Epoch 245][40/66] loss=0.771906 avg=0.853956 VRAM=39.2GiB | 81.9% done | ETA(epoch): 502s + [Epoch 245][50/66] loss=0.868903 avg=0.847144 VRAM=39.5GiB | 81.9% done | ETA(epoch): 309s + [Epoch 245][60/66] loss=0.853665 avg=0.853279 VRAM=39.2GiB | 82.0% done | ETA(epoch): 116s + Train loss: 0.856024 (1272.9s) ETA: 1263min + Val loss: 0.883848 [t_0.0-0.2=1.0820 t_0.2-0.4=0.9888 t_0.4-0.6=0.8491 t_0.6-0.8=0.7069 t_0.8-1.0=0.6562] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0245 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 246/299 (82% done) --- + [Epoch 246][10/66] loss=0.845169 avg=0.869421 VRAM=39.5GiB | 82.1% done | ETA(epoch): 1079s + [Epoch 246][20/66] loss=0.699208 avg=0.876130 VRAM=39.2GiB | 82.1% done | ETA(epoch): 886s + [Epoch 246][30/66] loss=0.853972 avg=0.857726 VRAM=39.5GiB | 82.2% done | ETA(epoch): 695s + [Epoch 246][40/66] loss=0.859749 avg=0.856377 VRAM=39.2GiB | 82.2% done | ETA(epoch): 502s + [Epoch 246][50/66] loss=0.904238 avg=0.860271 VRAM=39.5GiB | 82.3% done | ETA(epoch): 309s + [Epoch 246][60/66] loss=0.883616 avg=0.858045 VRAM=39.2GiB | 82.3% done | ETA(epoch): 116s + Train loss: 0.853930 (1273.4s) ETA: 1240min + Val loss: 0.825267 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0096 t_0.4-0.6=0.8103 t_0.6-0.8=0.7117 t_0.8-1.0=0.6612] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 247/299 (82% done) --- + [Epoch 247][10/66] loss=0.838434 avg=0.877118 VRAM=39.5GiB | 82.4% done | ETA(epoch): 1077s + [Epoch 247][20/66] loss=0.854636 avg=0.862571 VRAM=39.2GiB | 82.4% done | ETA(epoch): 886s + [Epoch 247][30/66] loss=0.891086 avg=0.862122 VRAM=39.5GiB | 82.5% done | ETA(epoch): 694s + [Epoch 247][40/66] loss=0.853634 avg=0.854436 VRAM=39.2GiB | 82.5% done | ETA(epoch): 501s + [Epoch 247][50/66] loss=0.796237 avg=0.853969 VRAM=39.5GiB | 82.6% done | ETA(epoch): 308s + [Epoch 247][60/66] loss=0.899426 avg=0.853125 VRAM=39.2GiB | 82.6% done | ETA(epoch): 116s + Train loss: 0.853478 (1271.9s) ETA: 1216min + Val loss: 0.853346 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9668 t_0.4-0.6=0.8668 t_0.6-0.8=0.7024 t_0.8-1.0=0.6663] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 248/299 (83% done) --- + [Epoch 248][10/66] loss=0.899701 avg=0.854784 VRAM=39.5GiB | 82.7% done | ETA(epoch): 1077s + [Epoch 248][20/66] loss=0.682114 avg=0.841388 VRAM=39.2GiB | 82.8% done | ETA(epoch): 885s + [Epoch 248][30/66] loss=0.749883 avg=0.835350 VRAM=39.5GiB | 82.8% done | ETA(epoch): 693s + [Epoch 248][40/66] loss=0.793340 avg=0.835260 VRAM=39.2GiB | 82.9% done | ETA(epoch): 501s + [Epoch 248][50/66] loss=0.891355 avg=0.836180 VRAM=39.5GiB | 82.9% done | ETA(epoch): 308s + [Epoch 248][60/66] loss=0.977160 avg=0.837159 VRAM=39.2GiB | 83.0% done | ETA(epoch): 116s + Train loss: 0.838128 (1271.4s) ETA: 1193min + Val loss: 0.841397 [t_0.0-0.2=1.0798 t_0.2-0.4=1.0027 t_0.4-0.6=0.8421 t_0.6-0.8=0.7172 t_0.8-1.0=0.6466] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0248 + Deleted old checkpoint: checkpoint_epoch_0245 +[MEM @ epoch 248 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 249/299 (83% done) --- + [Epoch 249][10/66] loss=0.857963 avg=0.886303 VRAM=39.5GiB | 83.1% done | ETA(epoch): 1077s + [Epoch 249][20/66] loss=0.824378 avg=0.866518 VRAM=39.2GiB | 83.1% done | ETA(epoch): 886s + [Epoch 249][30/66] loss=0.810444 avg=0.864420 VRAM=39.5GiB | 83.2% done | ETA(epoch): 693s + [Epoch 249][40/66] loss=0.718660 avg=0.865582 VRAM=39.2GiB | 83.2% done | ETA(epoch): 502s + [Epoch 249][50/66] loss=0.896349 avg=0.864715 VRAM=39.5GiB | 83.3% done | ETA(epoch): 309s + [Epoch 249][60/66] loss=0.808256 avg=0.859884 VRAM=39.2GiB | 83.3% done | ETA(epoch): 116s + Train loss: 0.862698 (1272.9s) ETA: 1169min + Val loss: 0.881448 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9879 t_0.4-0.6=0.8538 t_0.6-0.8=0.7265 t_0.8-1.0=0.6483] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 250/299 (83% done) --- + [Epoch 250][10/66] loss=0.882286 avg=0.833482 VRAM=39.5GiB | 83.4% done | ETA(epoch): 1076s + [Epoch 250][20/66] loss=0.956815 avg=0.830259 VRAM=39.2GiB | 83.4% done | ETA(epoch): 885s + [Epoch 250][30/66] loss=0.839778 avg=0.842098 VRAM=39.5GiB | 83.5% done | ETA(epoch): 693s + [Epoch 250][40/66] loss=0.953521 avg=0.855956 VRAM=39.2GiB | 83.5% done | ETA(epoch): 501s + [Epoch 250][50/66] loss=0.861472 avg=0.852410 VRAM=39.5GiB | 83.6% done | ETA(epoch): 308s + [Epoch 250][60/66] loss=0.902159 avg=0.855773 VRAM=39.2GiB | 83.6% done | ETA(epoch): 116s + Train loss: 0.856607 (1271.7s) ETA: 1146min + Val loss: 0.852616 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0240 t_0.4-0.6=0.8611 t_0.6-0.8=0.7186 t_0.8-1.0=0.6588] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 251/299 (84% done) --- + [Epoch 251][10/66] loss=0.948632 avg=0.847434 VRAM=39.5GiB | 83.7% done | ETA(epoch): 1080s + [Epoch 251][20/66] loss=0.762473 avg=0.859221 VRAM=39.2GiB | 83.8% done | ETA(epoch): 891s + [Epoch 251][30/66] loss=0.896175 avg=0.849715 VRAM=39.5GiB | 83.8% done | ETA(epoch): 696s + [Epoch 251][40/66] loss=0.800812 avg=0.854907 VRAM=39.2GiB | 83.9% done | ETA(epoch): 502s + [Epoch 251][50/66] loss=0.905560 avg=0.858122 VRAM=39.5GiB | 83.9% done | ETA(epoch): 309s + [Epoch 251][60/66] loss=0.860152 avg=0.854915 VRAM=39.2GiB | 84.0% done | ETA(epoch): 116s + Train loss: 0.857512 (1273.6s) ETA: 1122min + Val loss: 0.875487 [t_0.0-0.2=1.0785 t_0.2-0.4=0.9831 t_0.4-0.6=0.8187 t_0.6-0.8=0.7378 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 252/299 (84% done) --- + [Epoch 252][10/66] loss=0.790944 avg=0.853102 VRAM=39.5GiB | 84.1% done | ETA(epoch): 1100s + [Epoch 252][20/66] loss=0.822467 avg=0.857445 VRAM=39.2GiB | 84.1% done | ETA(epoch): 894s + [Epoch 252][30/66] loss=0.812747 avg=0.864819 VRAM=39.5GiB | 84.2% done | ETA(epoch): 698s + [Epoch 252][40/66] loss=0.909432 avg=0.866397 VRAM=39.2GiB | 84.2% done | ETA(epoch): 503s + [Epoch 252][50/66] loss=0.815282 avg=0.861181 VRAM=39.5GiB | 84.3% done | ETA(epoch): 309s + [Epoch 252][60/66] loss=0.816426 avg=0.862019 VRAM=39.2GiB | 84.3% done | ETA(epoch): 116s + Train loss: 0.857471 (1275.6s) ETA: 1099min + Val loss: 0.852936 [t_0.0-0.2=1.0722 t_0.2-0.4=1.0159 t_0.4-0.6=0.8663 t_0.6-0.8=0.6885 t_0.8-1.0=0.6580] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 253/299 (84% done) --- + [Epoch 253][10/66] loss=0.732246 avg=0.846307 VRAM=39.5GiB | 84.4% done | ETA(epoch): 1078s + [Epoch 253][20/66] loss=0.939244 avg=0.860247 VRAM=39.2GiB | 84.4% done | ETA(epoch): 886s + [Epoch 253][30/66] loss=0.896094 avg=0.866388 VRAM=39.5GiB | 84.5% done | ETA(epoch): 693s + [Epoch 253][40/66] loss=0.751526 avg=0.865268 VRAM=39.2GiB | 84.5% done | ETA(epoch): 501s + [Epoch 253][50/66] loss=0.766416 avg=0.864573 VRAM=39.5GiB | 84.6% done | ETA(epoch): 308s + [Epoch 253][60/66] loss=0.751991 avg=0.859900 VRAM=39.2GiB | 84.6% done | ETA(epoch): 116s + Train loss: 0.860780 (1271.6s) ETA: 1075min + Val loss: 0.852442 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9702 t_0.4-0.6=0.8379 t_0.6-0.8=0.7202 t_0.8-1.0=0.6754] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 254/299 (85% done) --- + [Epoch 254][10/66] loss=0.891066 avg=0.823579 VRAM=39.5GiB | 84.7% done | ETA(epoch): 1079s + [Epoch 254][20/66] loss=0.881021 avg=0.832609 VRAM=39.2GiB | 84.8% done | ETA(epoch): 886s + [Epoch 254][30/66] loss=0.771729 avg=0.834963 VRAM=39.5GiB | 84.8% done | ETA(epoch): 693s + [Epoch 254][40/66] loss=0.784967 avg=0.836494 VRAM=39.2GiB | 84.9% done | ETA(epoch): 501s + [Epoch 254][50/66] loss=0.922898 avg=0.845518 VRAM=39.5GiB | 84.9% done | ETA(epoch): 308s + [Epoch 254][60/66] loss=0.981729 avg=0.843146 VRAM=39.2GiB | 85.0% done | ETA(epoch): 116s + Train loss: 0.844856 (1272.9s) ETA: 1052min + Val loss: 0.886367 [t_0.0-0.2=1.0845 t_0.2-0.4=0.9984 t_0.4-0.6=0.8020 t_0.6-0.8=0.7326 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 255/299 (85% done) --- + [Epoch 255][10/66] loss=0.736704 avg=0.849322 VRAM=39.5GiB | 85.1% done | ETA(epoch): 1078s + [Epoch 255][20/66] loss=0.903317 avg=0.855032 VRAM=39.2GiB | 85.1% done | ETA(epoch): 885s + [Epoch 255][30/66] loss=0.815576 avg=0.850507 VRAM=39.5GiB | 85.2% done | ETA(epoch): 693s + [Epoch 255][40/66] loss=0.948680 avg=0.847981 VRAM=39.2GiB | 85.2% done | ETA(epoch): 501s + [Epoch 255][50/66] loss=0.930077 avg=0.846964 VRAM=39.5GiB | 85.3% done | ETA(epoch): 308s + [Epoch 255][60/66] loss=0.768393 avg=0.849442 VRAM=39.2GiB | 85.3% done | ETA(epoch): 116s + Train loss: 0.857689 (1271.3s) ETA: 1028min + Val loss: 0.849763 [t_0.0-0.2=1.0782 t_0.2-0.4=0.9964 t_0.4-0.6=0.8248 t_0.6-0.8=0.7119 t_0.8-1.0=0.6653] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 256/299 (85% done) --- + [Epoch 256][10/66] loss=0.935567 avg=0.894206 VRAM=39.5GiB | 85.4% done | ETA(epoch): 1077s + [Epoch 256][20/66] loss=0.742144 avg=0.864137 VRAM=39.2GiB | 85.4% done | ETA(epoch): 885s + [Epoch 256][30/66] loss=0.888230 avg=0.849980 VRAM=39.5GiB | 85.5% done | ETA(epoch): 693s + [Epoch 256][40/66] loss=0.780552 avg=0.860741 VRAM=39.2GiB | 85.5% done | ETA(epoch): 500s + [Epoch 256][50/66] loss=0.878829 avg=0.863965 VRAM=39.5GiB | 85.6% done | ETA(epoch): 308s + [Epoch 256][60/66] loss=0.887207 avg=0.865095 VRAM=39.2GiB | 85.6% done | ETA(epoch): 116s + Train loss: 0.866542 (1270.7s) ETA: 1005min + Val loss: 0.844527 [t_0.0-0.2=1.0667 t_0.2-0.4=1.0012 t_0.4-0.6=0.8619 t_0.6-0.8=0.6832 t_0.8-1.0=0.6459] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 257/299 (86% done) --- + [Epoch 257][10/66] loss=0.907758 avg=0.893455 VRAM=39.5GiB | 85.7% done | ETA(epoch): 1078s + [Epoch 257][20/66] loss=0.811328 avg=0.875958 VRAM=39.2GiB | 85.8% done | ETA(epoch): 886s + [Epoch 257][30/66] loss=0.897093 avg=0.869174 VRAM=39.5GiB | 85.8% done | ETA(epoch): 694s + [Epoch 257][40/66] loss=0.881800 avg=0.874894 VRAM=39.2GiB | 85.9% done | ETA(epoch): 501s + [Epoch 257][50/66] loss=0.832277 avg=0.870924 VRAM=39.5GiB | 85.9% done | ETA(epoch): 308s + [Epoch 257][60/66] loss=0.851659 avg=0.870163 VRAM=39.2GiB | 86.0% done | ETA(epoch): 116s + Train loss: 0.872756 (1272.8s) ETA: 981min + Val loss: 0.847687 [t_0.0-0.2=1.0793 t_0.2-0.4=0.9473 t_0.4-0.6=0.8748 t_0.6-0.8=0.7036 t_0.8-1.0=0.6593] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 258/299 (86% done) --- + [Epoch 258][10/66] loss=0.759696 avg=0.846860 VRAM=39.5GiB | 86.1% done | ETA(epoch): 1078s + [Epoch 258][20/66] loss=0.915543 avg=0.873901 VRAM=39.2GiB | 86.1% done | ETA(epoch): 885s + [Epoch 258][30/66] loss=0.882741 avg=0.865178 VRAM=39.5GiB | 86.2% done | ETA(epoch): 692s + [Epoch 258][40/66] loss=0.867005 avg=0.863034 VRAM=39.2GiB | 86.2% done | ETA(epoch): 500s + [Epoch 258][50/66] loss=0.848023 avg=0.857388 VRAM=39.5GiB | 86.3% done | ETA(epoch): 308s + [Epoch 258][60/66] loss=0.944816 avg=0.849755 VRAM=39.2GiB | 86.3% done | ETA(epoch): 115s + Train loss: 0.853408 (1270.2s) ETA: 958min + Val loss: 0.860034 [t_0.0-0.2=1.0875 t_0.2-0.4=1.0053 t_0.4-0.6=0.8310 t_0.6-0.8=0.7207 t_0.8-1.0=0.6489] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 259/299 (86% done) --- + [Epoch 259][10/66] loss=0.857248 avg=0.866828 VRAM=39.5GiB | 86.4% done | ETA(epoch): 1079s + [Epoch 259][20/66] loss=0.740967 avg=0.842407 VRAM=39.2GiB | 86.4% done | ETA(epoch): 886s + [Epoch 259][30/66] loss=0.871348 avg=0.836874 VRAM=39.5GiB | 86.5% done | ETA(epoch): 693s + [Epoch 259][40/66] loss=0.882503 avg=0.836697 VRAM=39.2GiB | 86.5% done | ETA(epoch): 501s + [Epoch 259][50/66] loss=0.878065 avg=0.845617 VRAM=39.5GiB | 86.6% done | ETA(epoch): 308s + [Epoch 259][60/66] loss=0.880955 avg=0.849816 VRAM=39.2GiB | 86.6% done | ETA(epoch): 116s + Train loss: 0.857268 (1271.3s) ETA: 934min + Val loss: 0.834478 [t_0.0-0.2=1.0605 t_0.2-0.4=0.9887 t_0.4-0.6=0.8603 t_0.6-0.8=0.7065 t_0.8-1.0=0.6553] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 260/299 (87% done) --- + [Epoch 260][10/66] loss=0.814895 avg=0.832893 VRAM=39.5GiB | 86.7% done | ETA(epoch): 1077s + [Epoch 260][20/66] loss=0.893971 avg=0.841526 VRAM=39.2GiB | 86.8% done | ETA(epoch): 885s + [Epoch 260][30/66] loss=0.721806 avg=0.845534 VRAM=39.5GiB | 86.8% done | ETA(epoch): 693s + [Epoch 260][40/66] loss=0.971198 avg=0.854681 VRAM=39.2GiB | 86.9% done | ETA(epoch): 500s + [Epoch 260][50/66] loss=0.950517 avg=0.855933 VRAM=39.5GiB | 86.9% done | ETA(epoch): 308s + [Epoch 260][60/66] loss=0.839356 avg=0.858517 VRAM=39.2GiB | 87.0% done | ETA(epoch): 115s + Train loss: 0.856789 (1273.1s) ETA: 911min + Val loss: 0.859275 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9987 t_0.4-0.6=0.8531 t_0.6-0.8=0.7363 t_0.8-1.0=0.6450] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 261/299 (87% done) --- + [Epoch 261][10/66] loss=0.792903 avg=0.846755 VRAM=39.5GiB | 87.1% done | ETA(epoch): 1081s + [Epoch 261][20/66] loss=0.935302 avg=0.860749 VRAM=39.2GiB | 87.1% done | ETA(epoch): 887s + [Epoch 261][30/66] loss=0.990263 avg=0.858560 VRAM=39.5GiB | 87.2% done | ETA(epoch): 694s + [Epoch 261][40/66] loss=0.819425 avg=0.867111 VRAM=39.2GiB | 87.2% done | ETA(epoch): 501s + [Epoch 261][50/66] loss=0.794799 avg=0.860404 VRAM=39.5GiB | 87.3% done | ETA(epoch): 308s + [Epoch 261][60/66] loss=0.893842 avg=0.859471 VRAM=39.2GiB | 87.3% done | ETA(epoch): 116s + Train loss: 0.862809 (1271.7s) ETA: 887min + Val loss: 0.853856 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9910 t_0.4-0.6=0.8452 t_0.6-0.8=0.7160 t_0.8-1.0=0.6583] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 262/299 (87% done) --- + [Epoch 262][10/66] loss=0.705563 avg=0.792951 VRAM=39.5GiB | 87.4% done | ETA(epoch): 1076s + [Epoch 262][20/66] loss=0.824569 avg=0.837110 VRAM=39.2GiB | 87.4% done | ETA(epoch): 885s + [Epoch 262][30/66] loss=0.814002 avg=0.835310 VRAM=39.5GiB | 87.5% done | ETA(epoch): 693s + [Epoch 262][40/66] loss=0.943607 avg=0.839056 VRAM=39.2GiB | 87.5% done | ETA(epoch): 500s + [Epoch 262][50/66] loss=0.794939 avg=0.841984 VRAM=39.5GiB | 87.6% done | ETA(epoch): 309s + [Epoch 262][60/66] loss=0.917295 avg=0.851556 VRAM=39.2GiB | 87.6% done | ETA(epoch): 116s + Train loss: 0.847425 (1274.2s) ETA: 864min + Val loss: 0.849113 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9899 t_0.4-0.6=0.8374 t_0.6-0.8=0.7266 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 263/299 (88% done) --- + [Epoch 263][10/66] loss=0.723506 avg=0.819003 VRAM=39.5GiB | 87.7% done | ETA(epoch): 1079s + [Epoch 263][20/66] loss=0.828667 avg=0.841853 VRAM=39.2GiB | 87.8% done | ETA(epoch): 886s + [Epoch 263][30/66] loss=0.889730 avg=0.850654 VRAM=39.5GiB | 87.8% done | ETA(epoch): 694s + [Epoch 263][40/66] loss=0.728897 avg=0.844666 VRAM=39.2GiB | 87.9% done | ETA(epoch): 503s + [Epoch 263][50/66] loss=0.764194 avg=0.841246 VRAM=39.5GiB | 87.9% done | ETA(epoch): 309s + [Epoch 263][60/66] loss=0.919710 avg=0.841197 VRAM=39.2GiB | 88.0% done | ETA(epoch): 116s + Train loss: 0.843965 (1275.3s) ETA: 841min + Val loss: 0.848117 [t_0.0-0.2=1.0762 t_0.2-0.4=0.9840 t_0.4-0.6=0.8262 t_0.6-0.8=0.7143 t_0.8-1.0=0.6621] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 264/299 (88% done) --- + [Epoch 264][10/66] loss=0.847889 avg=0.843523 VRAM=39.5GiB | 88.1% done | ETA(epoch): 1080s + [Epoch 264][20/66] loss=0.730262 avg=0.845044 VRAM=39.2GiB | 88.1% done | ETA(epoch): 888s + [Epoch 264][30/66] loss=0.898023 avg=0.842528 VRAM=39.5GiB | 88.2% done | ETA(epoch): 695s + [Epoch 264][40/66] loss=0.885985 avg=0.857597 VRAM=39.2GiB | 88.2% done | ETA(epoch): 503s + [Epoch 264][50/66] loss=0.654226 avg=0.855058 VRAM=39.5GiB | 88.3% done | ETA(epoch): 309s + [Epoch 264][60/66] loss=0.898017 avg=0.856466 VRAM=39.2GiB | 88.3% done | ETA(epoch): 116s + Train loss: 0.856068 (1274.1s) ETA: 817min + Val loss: 0.849759 [t_0.0-0.2=1.0662 t_0.2-0.4=0.9642 t_0.4-0.6=0.8371 t_0.6-0.8=0.7327 t_0.8-1.0=0.6592] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 265/299 (88% done) --- + [Epoch 265][10/66] loss=0.931541 avg=0.866600 VRAM=39.5GiB | 88.4% done | ETA(epoch): 1077s + [Epoch 265][20/66] loss=0.822381 avg=0.867102 VRAM=39.2GiB | 88.4% done | ETA(epoch): 884s + [Epoch 265][30/66] loss=0.956446 avg=0.866787 VRAM=39.5GiB | 88.5% done | ETA(epoch): 692s + [Epoch 265][40/66] loss=0.861115 avg=0.871152 VRAM=39.2GiB | 88.5% done | ETA(epoch): 500s + [Epoch 265][50/66] loss=0.914905 avg=0.863020 VRAM=39.5GiB | 88.6% done | ETA(epoch): 308s + [Epoch 265][60/66] loss=0.817615 avg=0.860182 VRAM=39.2GiB | 88.6% done | ETA(epoch): 115s + Train loss: 0.856454 (1268.8s) ETA: 794min + Val loss: 0.855828 [t_0.0-0.2=1.0769 t_0.2-0.4=1.0036 t_0.4-0.6=0.8018 t_0.6-0.8=0.7073 t_0.8-1.0=0.6497] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 266/299 (89% done) --- + [Epoch 266][10/66] loss=0.823201 avg=0.864657 VRAM=39.5GiB | 88.7% done | ETA(epoch): 1077s + [Epoch 266][20/66] loss=0.814104 avg=0.854942 VRAM=39.2GiB | 88.8% done | ETA(epoch): 885s + [Epoch 266][30/66] loss=0.948177 avg=0.858659 VRAM=39.5GiB | 88.8% done | ETA(epoch): 692s + [Epoch 266][40/66] loss=0.935248 avg=0.862909 VRAM=39.2GiB | 88.9% done | ETA(epoch): 500s + [Epoch 266][50/66] loss=0.744268 avg=0.855816 VRAM=39.5GiB | 88.9% done | ETA(epoch): 308s + [Epoch 266][60/66] loss=0.787123 avg=0.851152 VRAM=39.2GiB | 89.0% done | ETA(epoch): 116s + Train loss: 0.854275 (1270.8s) ETA: 770min + Val loss: 0.868577 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9668 t_0.4-0.6=0.8436 t_0.6-0.8=0.7162 t_0.8-1.0=0.6505] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 267/299 (89% done) --- + [Epoch 267][10/66] loss=0.815486 avg=0.821806 VRAM=39.5GiB | 89.1% done | ETA(epoch): 1092s + [Epoch 267][20/66] loss=0.871962 avg=0.840470 VRAM=39.2GiB | 89.1% done | ETA(epoch): 890s + [Epoch 267][30/66] loss=0.899920 avg=0.848761 VRAM=39.5GiB | 89.2% done | ETA(epoch): 695s + [Epoch 267][40/66] loss=0.933449 avg=0.860041 VRAM=39.2GiB | 89.2% done | ETA(epoch): 502s + [Epoch 267][50/66] loss=0.853497 avg=0.856886 VRAM=39.5GiB | 89.3% done | ETA(epoch): 309s + [Epoch 267][60/66] loss=0.874542 avg=0.854003 VRAM=39.2GiB | 89.3% done | ETA(epoch): 116s + Train loss: 0.851351 (1272.3s) ETA: 747min + Val loss: 0.877084 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9896 t_0.4-0.6=0.8547 t_0.6-0.8=0.7291 t_0.8-1.0=0.6575] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 268/299 (89% done) --- + [Epoch 268][10/66] loss=0.808588 avg=0.856121 VRAM=39.5GiB | 89.4% done | ETA(epoch): 1077s + [Epoch 268][20/66] loss=0.848329 avg=0.855468 VRAM=39.2GiB | 89.4% done | ETA(epoch): 885s + [Epoch 268][30/66] loss=0.889877 avg=0.859805 VRAM=39.5GiB | 89.5% done | ETA(epoch): 693s + [Epoch 268][40/66] loss=0.778230 avg=0.857842 VRAM=39.2GiB | 89.5% done | ETA(epoch): 501s + [Epoch 268][50/66] loss=0.859113 avg=0.862087 VRAM=39.5GiB | 89.6% done | ETA(epoch): 308s + [Epoch 268][60/66] loss=0.854041 avg=0.858868 VRAM=39.2GiB | 89.6% done | ETA(epoch): 115s + Train loss: 0.859972 (1270.6s) ETA: 724min + Val loss: 0.869478 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9657 t_0.4-0.6=0.8665 t_0.6-0.8=0.6846 t_0.8-1.0=0.6600] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 269/299 (90% done) --- + [Epoch 269][10/66] loss=0.891709 avg=0.840253 VRAM=39.5GiB | 89.7% done | ETA(epoch): 1077s + [Epoch 269][20/66] loss=0.995270 avg=0.851972 VRAM=39.2GiB | 89.8% done | ETA(epoch): 885s + [Epoch 269][30/66] loss=0.932956 avg=0.856887 VRAM=39.5GiB | 89.8% done | ETA(epoch): 693s + [Epoch 269][40/66] loss=0.872032 avg=0.866715 VRAM=39.2GiB | 89.9% done | ETA(epoch): 500s + [Epoch 269][50/66] loss=0.838738 avg=0.868591 VRAM=39.5GiB | 89.9% done | ETA(epoch): 308s + [Epoch 269][60/66] loss=0.869135 avg=0.868660 VRAM=39.2GiB | 90.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 17819 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 17819 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 17819 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 17819 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 17819 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 17819 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 17819 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 17819 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 17819 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 17819 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + Train loss: 0.870781 (2304.2s) ETA: 702min + Val loss: 0.866258 [t_0.0-0.2=1.0642 t_0.2-0.4=1.0146 t_0.4-0.6=0.8302 t_0.6-0.8=0.7240 t_0.8-1.0=0.6641] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 270/299 (90% done) --- + [Epoch 270][10/66] loss=0.883219 avg=0.888616 VRAM=39.5GiB | 90.1% done | ETA(epoch): 1075s + [Epoch 270][20/66] loss=0.911662 avg=0.886172 VRAM=39.2GiB | 90.1% done | ETA(epoch): 885s + [Epoch 270][30/66] loss=0.858314 avg=0.880009 VRAM=39.5GiB | 90.2% done | ETA(epoch): 692s + [Epoch 270][40/66] loss=0.869510 avg=0.873906 VRAM=39.2GiB | 90.2% done | ETA(epoch): 500s + [Epoch 270][50/66] loss=0.783256 avg=0.868082 VRAM=39.5GiB | 90.3% done | ETA(epoch): 308s + [Epoch 270][60/66] loss=0.944966 avg=0.862731 VRAM=39.2GiB | 90.3% done | ETA(epoch): 116s + Train loss: 0.860189 (1272.2s) ETA: 679min + Val loss: 0.865460 [t_0.0-0.2=1.0855 t_0.2-0.4=1.0022 t_0.4-0.6=0.8581 t_0.6-0.8=0.7155 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 271/299 (90% done) --- + [Epoch 271][10/66] loss=0.875512 avg=0.837475 VRAM=39.5GiB | 90.4% done | ETA(epoch): 1079s + [Epoch 271][20/66] loss=0.765806 avg=0.841569 VRAM=39.2GiB | 90.4% done | ETA(epoch): 886s + [Epoch 271][30/66] loss=0.781524 avg=0.843830 VRAM=39.5GiB | 90.5% done | ETA(epoch): 693s + [Epoch 271][40/66] loss=0.848330 avg=0.845851 VRAM=39.2GiB | 90.5% done | ETA(epoch): 501s + [Epoch 271][50/66] loss=0.844246 avg=0.850392 VRAM=39.5GiB | 90.6% done | ETA(epoch): 309s + [Epoch 271][60/66] loss=0.895334 avg=0.859212 VRAM=39.2GiB | 90.6% done | ETA(epoch): 116s + Train loss: 0.861908 (1273.4s) ETA: 655min + Val loss: 0.842980 [t_0.0-0.2=1.0918 t_0.2-0.4=0.9930 t_0.4-0.6=0.8207 t_0.6-0.8=0.7059 t_0.8-1.0=0.6364] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 272/299 (91% done) --- + [Epoch 272][10/66] loss=0.897885 avg=0.859120 VRAM=39.5GiB | 90.7% done | ETA(epoch): 1077s + [Epoch 272][20/66] loss=0.947325 avg=0.848382 VRAM=39.2GiB | 90.8% done | ETA(epoch): 885s + [Epoch 272][30/66] loss=0.890120 avg=0.857964 VRAM=39.5GiB | 90.8% done | ETA(epoch): 693s + [Epoch 272][40/66] loss=0.945337 avg=0.860851 VRAM=39.2GiB | 90.9% done | ETA(epoch): 500s + [Epoch 272][50/66] loss=0.748927 avg=0.859013 VRAM=39.5GiB | 90.9% done | ETA(epoch): 308s + [Epoch 272][60/66] loss=0.954295 avg=0.859209 VRAM=39.2GiB | 91.0% done | ETA(epoch): 116s + Train loss: 0.859105 (1271.0s) ETA: 632min + Val loss: 0.866034 [t_0.0-0.2=1.0959 t_0.2-0.4=0.9215 t_0.4-0.6=0.8398 t_0.6-0.8=0.7423 t_0.8-1.0=0.6672] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 273/299 (91% done) --- + [Epoch 273][10/66] loss=0.806742 avg=0.837807 VRAM=39.5GiB | 91.1% done | ETA(epoch): 1080s + [Epoch 273][20/66] loss=0.736313 avg=0.824580 VRAM=39.2GiB | 91.1% done | ETA(epoch): 887s + [Epoch 273][30/66] loss=0.769799 avg=0.826563 VRAM=39.5GiB | 91.2% done | ETA(epoch): 694s + [Epoch 273][40/66] loss=0.809993 avg=0.836888 VRAM=39.2GiB | 91.2% done | ETA(epoch): 501s + [Epoch 273][50/66] loss=0.868087 avg=0.843867 VRAM=39.5GiB | 91.3% done | ETA(epoch): 308s + [Epoch 273][60/66] loss=0.993380 avg=0.852830 VRAM=39.2GiB | 91.3% done | ETA(epoch): 116s + Train loss: 0.856642 (1270.5s) ETA: 608min + Val loss: 0.830192 [t_0.0-0.2=1.0777 t_0.2-0.4=0.9765 t_0.4-0.6=0.8331 t_0.6-0.8=0.7003 t_0.8-1.0=0.6552] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 274/299 (91% done) --- + [Epoch 274][10/66] loss=0.805146 avg=0.866492 VRAM=39.5GiB | 91.4% done | ETA(epoch): 1077s + [Epoch 274][20/66] loss=0.849647 avg=0.854842 VRAM=39.2GiB | 91.4% done | ETA(epoch): 885s + [Epoch 274][30/66] loss=0.869915 avg=0.845520 VRAM=39.5GiB | 91.5% done | ETA(epoch): 693s + [Epoch 274][40/66] loss=0.744982 avg=0.848216 VRAM=39.2GiB | 91.5% done | ETA(epoch): 500s + [Epoch 274][50/66] loss=0.811647 avg=0.845441 VRAM=39.5GiB | 91.6% done | ETA(epoch): 308s + [Epoch 274][60/66] loss=0.946158 avg=0.857706 VRAM=39.2GiB | 91.6% done | ETA(epoch): 115s + Train loss: 0.855269 (1268.8s) ETA: 585min + Val loss: 0.852132 [t_0.0-0.2=1.0684 t_0.2-0.4=0.9640 t_0.4-0.6=0.8174 t_0.6-0.8=0.7311 t_0.8-1.0=0.6485] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 275/299 (92% done) --- + [Epoch 275][10/66] loss=0.981441 avg=0.884699 VRAM=39.5GiB | 91.7% done | ETA(epoch): 1089s + [Epoch 275][20/66] loss=0.954464 avg=0.854296 VRAM=39.2GiB | 91.8% done | ETA(epoch): 890s + [Epoch 275][30/66] loss=0.742476 avg=0.842405 VRAM=39.5GiB | 91.8% done | ETA(epoch): 695s + [Epoch 275][40/66] loss=0.827348 avg=0.851785 VRAM=39.2GiB | 91.9% done | ETA(epoch): 502s + [Epoch 275][50/66] loss=0.816387 avg=0.857243 VRAM=39.5GiB | 91.9% done | ETA(epoch): 309s + [Epoch 275][60/66] loss=0.927205 avg=0.863859 VRAM=39.2GiB | 92.0% done | ETA(epoch): 116s + Train loss: 0.864365 (1272.3s) ETA: 561min + Val loss: 0.861171 [t_0.0-0.2=1.0626 t_0.2-0.4=0.9873 t_0.4-0.6=0.8487 t_0.6-0.8=0.7206 t_0.8-1.0=0.6533] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 276/299 (92% done) --- + [Epoch 276][10/66] loss=0.905104 avg=0.900188 VRAM=39.5GiB | 92.1% done | ETA(epoch): 1099s + [Epoch 276][20/66] loss=0.878607 avg=0.867047 VRAM=39.2GiB | 92.1% done | ETA(epoch): 894s + [Epoch 276][30/66] loss=0.982136 avg=0.860693 VRAM=39.5GiB | 92.2% done | ETA(epoch): 697s + [Epoch 276][40/66] loss=0.865258 avg=0.860097 VRAM=39.2GiB | 92.2% done | ETA(epoch): 503s + [Epoch 276][50/66] loss=1.028563 avg=0.858513 VRAM=39.5GiB | 92.3% done | ETA(epoch): 309s + [Epoch 276][60/66] loss=0.839629 avg=0.865986 VRAM=39.2GiB | 92.3% done | ETA(epoch): 116s + Train loss: 0.867679 (1275.6s) ETA: 538min + Val loss: 0.862946 [t_0.0-0.2=1.0660 t_0.2-0.4=1.0048 t_0.4-0.6=0.8675 t_0.6-0.8=0.7001 t_0.8-1.0=0.6699] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 277/299 (92% done) --- + [Epoch 277][10/66] loss=0.840841 avg=0.845051 VRAM=39.5GiB | 92.4% done | ETA(epoch): 1076s + [Epoch 277][20/66] loss=0.769123 avg=0.847751 VRAM=39.2GiB | 92.4% done | ETA(epoch): 884s + [Epoch 277][30/66] loss=0.860221 avg=0.860504 VRAM=39.5GiB | 92.5% done | ETA(epoch): 692s + [Epoch 277][40/66] loss=0.891082 avg=0.859024 VRAM=39.2GiB | 92.5% done | ETA(epoch): 500s + [Epoch 277][50/66] loss=0.749953 avg=0.851793 VRAM=39.5GiB | 92.6% done | ETA(epoch): 308s + [Epoch 277][60/66] loss=0.895431 avg=0.853830 VRAM=39.2GiB | 92.6% done | ETA(epoch): 116s + Train loss: 0.855863 (1271.4s) ETA: 514min + Val loss: 0.841665 [t_0.0-0.2=1.0614 t_0.2-0.4=1.0028 t_0.4-0.6=0.8372 t_0.6-0.8=0.7213 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 278/299 (93% done) --- + [Epoch 278][10/66] loss=0.849717 avg=0.878970 VRAM=39.5GiB | 92.7% done | ETA(epoch): 1075s + [Epoch 278][20/66] loss=0.816244 avg=0.877404 VRAM=39.2GiB | 92.8% done | ETA(epoch): 885s + [Epoch 278][30/66] loss=0.986211 avg=0.871222 VRAM=39.5GiB | 92.8% done | ETA(epoch): 692s + [Epoch 278][40/66] loss=0.932109 avg=0.868364 VRAM=39.2GiB | 92.9% done | ETA(epoch): 500s + [Epoch 278][50/66] loss=0.835148 avg=0.870674 VRAM=39.5GiB | 92.9% done | ETA(epoch): 308s + [Epoch 278][60/66] loss=0.921790 avg=0.867435 VRAM=39.2GiB | 93.0% done | ETA(epoch): 115s + Train loss: 0.867868 (1270.7s) ETA: 491min + Val loss: 0.866282 [t_0.0-0.2=1.0391 t_0.2-0.4=0.9875 t_0.4-0.6=0.8571 t_0.6-0.8=0.7191 t_0.8-1.0=0.6483] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 279/299 (93% done) --- + [Epoch 279][10/66] loss=0.850762 avg=0.840889 VRAM=39.5GiB | 93.1% done | ETA(epoch): 1077s + [Epoch 279][20/66] loss=0.889750 avg=0.855547 VRAM=39.2GiB | 93.1% done | ETA(epoch): 887s + [Epoch 279][30/66] loss=0.939675 avg=0.874471 VRAM=39.5GiB | 93.2% done | ETA(epoch): 694s + [Epoch 279][40/66] loss=0.880745 avg=0.871015 VRAM=39.2GiB | 93.2% done | ETA(epoch): 504s + [Epoch 279][50/66] loss=0.845855 avg=0.863316 VRAM=39.5GiB | 93.3% done | ETA(epoch): 310s + [Epoch 279][60/66] loss=0.914427 avg=0.857972 VRAM=39.2GiB | 93.3% done | ETA(epoch): 116s + Train loss: 0.857925 (1275.2s) ETA: 468min + Val loss: 0.830108 [t_0.0-0.2=1.0932 t_0.2-0.4=1.0104 t_0.4-0.6=0.7891 t_0.6-0.8=0.7211 t_0.8-1.0=0.6661] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 280/299 (93% done) --- + [Epoch 280][10/66] loss=0.814669 avg=0.831186 VRAM=39.5GiB | 93.4% done | ETA(epoch): 1076s + [Epoch 280][20/66] loss=0.806987 avg=0.834839 VRAM=39.2GiB | 93.4% done | ETA(epoch): 884s + [Epoch 280][30/66] loss=0.780859 avg=0.818369 VRAM=39.5GiB | 93.5% done | ETA(epoch): 693s + [Epoch 280][40/66] loss=0.839913 avg=0.830335 VRAM=39.2GiB | 93.5% done | ETA(epoch): 500s + [Epoch 280][50/66] loss=0.757559 avg=0.837086 VRAM=39.5GiB | 93.6% done | ETA(epoch): 308s + [Epoch 280][60/66] loss=0.788592 avg=0.838901 VRAM=39.2GiB | 93.6% done | ETA(epoch): 115s + Train loss: 0.846958 (1270.4s) ETA: 444min + Val loss: 0.830348 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9608 t_0.4-0.6=0.8235 t_0.6-0.8=0.7168 t_0.8-1.0=0.6768] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 281/299 (94% done) --- + [Epoch 281][10/66] loss=0.901106 avg=0.853776 VRAM=39.5GiB | 93.7% done | ETA(epoch): 1079s + [Epoch 281][20/66] loss=0.857201 avg=0.852867 VRAM=39.2GiB | 93.8% done | ETA(epoch): 885s + [Epoch 281][30/66] loss=0.902209 avg=0.854766 VRAM=39.5GiB | 93.8% done | ETA(epoch): 693s + [Epoch 281][40/66] loss=0.829233 avg=0.856737 VRAM=39.2GiB | 93.9% done | ETA(epoch): 500s + [Epoch 281][50/66] loss=0.892958 avg=0.862259 VRAM=39.5GiB | 93.9% done | ETA(epoch): 308s + [Epoch 281][60/66] loss=0.907544 avg=0.866758 VRAM=39.2GiB | 94.0% done | ETA(epoch): 116s + Train loss: 0.866620 (1271.0s) ETA: 421min + Val loss: 0.859253 [t_0.0-0.2=1.0578 t_0.2-0.4=0.9714 t_0.4-0.6=0.8497 t_0.6-0.8=0.7131 t_0.8-1.0=0.6557] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0281 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 282/299 (94% done) --- + [Epoch 282][10/66] loss=0.895281 avg=0.876488 VRAM=39.5GiB | 94.1% done | ETA(epoch): 1078s + [Epoch 282][20/66] loss=0.895941 avg=0.837646 VRAM=39.2GiB | 94.1% done | ETA(epoch): 885s + [Epoch 282][30/66] loss=0.809431 avg=0.844297 VRAM=39.5GiB | 94.2% done | ETA(epoch): 693s + [Epoch 282][40/66] loss=0.836059 avg=0.839626 VRAM=39.2GiB | 94.2% done | ETA(epoch): 500s + [Epoch 282][50/66] loss=0.905396 avg=0.842484 VRAM=39.5GiB | 94.3% done | ETA(epoch): 308s + [Epoch 282][60/66] loss=0.816389 avg=0.848781 VRAM=39.2GiB | 94.3% done | ETA(epoch): 115s + Train loss: 0.851334 (1269.8s) ETA: 397min + Val loss: 0.849561 [t_0.0-0.2=1.0690 t_0.2-0.4=0.9699 t_0.4-0.6=0.8314 t_0.6-0.8=0.7201 t_0.8-1.0=0.6638] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 283/299 (94% done) --- + [Epoch 283][10/66] loss=0.768276 avg=0.896250 VRAM=39.5GiB | 94.4% done | ETA(epoch): 1078s + [Epoch 283][20/66] loss=0.964185 avg=0.885317 VRAM=39.2GiB | 94.4% done | ETA(epoch): 886s + [Epoch 283][30/66] loss=0.842080 avg=0.881893 VRAM=39.5GiB | 94.5% done | ETA(epoch): 693s + [Epoch 283][40/66] loss=0.746583 avg=0.874948 VRAM=39.2GiB | 94.5% done | ETA(epoch): 500s + [Epoch 283][50/66] loss=0.839190 avg=0.873645 VRAM=39.5GiB | 94.6% done | ETA(epoch): 308s + [Epoch 283][60/66] loss=0.872759 avg=0.868019 VRAM=39.2GiB | 94.6% done | ETA(epoch): 115s + Train loss: 0.865858 (1270.1s) ETA: 374min + Val loss: 0.886696 [t_0.0-0.2=1.0719 t_0.2-0.4=0.9910 t_0.4-0.6=0.8473 t_0.6-0.8=0.7219 t_0.8-1.0=0.6434] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 284/299 (95% done) --- + [Epoch 284][10/66] loss=0.895537 avg=0.853894 VRAM=39.5GiB | 94.7% done | ETA(epoch): 1077s + [Epoch 284][20/66] loss=0.894342 avg=0.863486 VRAM=39.2GiB | 94.8% done | ETA(epoch): 885s + [Epoch 284][30/66] loss=0.930846 avg=0.846618 VRAM=39.5GiB | 94.8% done | ETA(epoch): 693s + [Epoch 284][40/66] loss=0.882666 avg=0.852740 VRAM=39.2GiB | 94.9% done | ETA(epoch): 500s + [Epoch 284][50/66] loss=0.930681 avg=0.863456 VRAM=39.5GiB | 94.9% done | ETA(epoch): 308s + [Epoch 284][60/66] loss=0.989453 avg=0.864832 VRAM=39.2GiB | 95.0% done | ETA(epoch): 116s + Train loss: 0.866687 (1270.7s) ETA: 351min + Val loss: 0.841470 [t_0.0-0.2=1.0822 t_0.2-0.4=0.9643 t_0.4-0.6=0.8477 t_0.6-0.8=0.7235 t_0.8-1.0=0.6635] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0284 + Deleted old checkpoint: checkpoint_epoch_0281 +[MEM @ epoch 284 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 285/299 (95% done) --- + [Epoch 285][10/66] loss=0.833017 avg=0.831185 VRAM=39.5GiB | 95.1% done | ETA(epoch): 1077s + [Epoch 285][20/66] loss=0.935538 avg=0.852517 VRAM=39.2GiB | 95.1% done | ETA(epoch): 885s + [Epoch 285][30/66] loss=0.878399 avg=0.850475 VRAM=39.5GiB | 95.2% done | ETA(epoch): 693s + [Epoch 285][40/66] loss=0.746365 avg=0.845560 VRAM=39.2GiB | 95.2% done | ETA(epoch): 500s + [Epoch 285][50/66] loss=0.701740 avg=0.846402 VRAM=39.5GiB | 95.3% done | ETA(epoch): 308s + [Epoch 285][60/66] loss=0.811983 avg=0.848922 VRAM=39.2GiB | 95.3% done | ETA(epoch): 115s + Train loss: 0.852494 (1270.3s) ETA: 327min + Val loss: 0.848832 [t_0.0-0.2=1.0878 t_0.2-0.4=1.0410 t_0.4-0.6=0.8758 t_0.6-0.8=0.6828 t_0.8-1.0=0.6669] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 286/299 (95% done) --- + [Epoch 286][10/66] loss=0.855674 avg=0.888500 VRAM=39.5GiB | 95.4% done | ETA(epoch): 1079s + [Epoch 286][20/66] loss=0.869657 avg=0.858922 VRAM=39.2GiB | 95.4% done | ETA(epoch): 886s + [Epoch 286][30/66] loss=0.795029 avg=0.855470 VRAM=39.5GiB | 95.5% done | ETA(epoch): 693s + [Epoch 286][40/66] loss=0.938262 avg=0.860155 VRAM=39.2GiB | 95.5% done | ETA(epoch): 500s + [Epoch 286][50/66] loss=0.818639 avg=0.860607 VRAM=39.5GiB | 95.6% done | ETA(epoch): 308s + [Epoch 286][60/66] loss=0.964940 avg=0.864875 VRAM=39.2GiB | 95.6% done | ETA(epoch): 115s + Train loss: 0.861164 (1269.8s) ETA: 304min + Val loss: 0.866572 [t_0.0-0.2=1.0859 t_0.2-0.4=1.0115 t_0.4-0.6=0.8349 t_0.6-0.8=0.7052 t_0.8-1.0=0.6555] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 287/299 (96% done) --- + [Epoch 287][10/66] loss=0.857126 avg=0.847116 VRAM=39.5GiB | 95.7% done | ETA(epoch): 1077s + [Epoch 287][20/66] loss=0.815061 avg=0.849654 VRAM=39.2GiB | 95.8% done | ETA(epoch): 885s + [Epoch 287][30/66] loss=0.859318 avg=0.855970 VRAM=39.5GiB | 95.8% done | ETA(epoch): 693s + [Epoch 287][40/66] loss=0.815872 avg=0.850431 VRAM=39.2GiB | 95.9% done | ETA(epoch): 500s + [Epoch 287][50/66] loss=0.868598 avg=0.855544 VRAM=39.5GiB | 95.9% done | ETA(epoch): 308s + [Epoch 287][60/66] loss=0.865888 avg=0.859375 VRAM=39.2GiB | 96.0% done | ETA(epoch): 115s + Train loss: 0.860413 (1269.8s) ETA: 280min + Val loss: 0.876628 [t_0.0-0.2=1.0926 t_0.2-0.4=0.9811 t_0.4-0.6=0.8630 t_0.6-0.8=0.7179 t_0.8-1.0=0.6549] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 288/299 (96% done) --- + [Epoch 288][10/66] loss=0.884156 avg=0.879494 VRAM=39.5GiB | 96.1% done | ETA(epoch): 1077s + [Epoch 288][20/66] loss=0.896132 avg=0.880915 VRAM=39.2GiB | 96.1% done | ETA(epoch): 884s + [Epoch 288][30/66] loss=0.952886 avg=0.880749 VRAM=39.5GiB | 96.2% done | ETA(epoch): 694s + [Epoch 288][40/66] loss=0.822124 avg=0.876174 VRAM=39.2GiB | 96.2% done | ETA(epoch): 501s + [Epoch 288][50/66] loss=0.820614 avg=0.867831 VRAM=39.5GiB | 96.3% done | ETA(epoch): 308s + [Epoch 288][60/66] loss=0.909203 avg=0.862423 VRAM=39.2GiB | 96.3% done | ETA(epoch): 116s + Train loss: 0.859615 (1270.8s) ETA: 257min + Val loss: 0.859631 [t_0.0-0.2=1.0712 t_0.2-0.4=0.9954 t_0.4-0.6=0.8532 t_0.6-0.8=0.7098 t_0.8-1.0=0.6443] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 289/299 (96% done) --- + [Epoch 289][10/66] loss=0.820084 avg=0.842580 VRAM=39.5GiB | 96.4% done | ETA(epoch): 1075s + [Epoch 289][20/66] loss=0.824299 avg=0.874024 VRAM=39.2GiB | 96.4% done | ETA(epoch): 884s + [Epoch 289][30/66] loss=0.979273 avg=0.870049 VRAM=39.5GiB | 96.5% done | ETA(epoch): 692s + [Epoch 289][40/66] loss=0.893286 avg=0.875596 VRAM=39.2GiB | 96.5% done | ETA(epoch): 500s + [Epoch 289][50/66] loss=0.705002 avg=0.871675 VRAM=39.5GiB | 96.6% done | ETA(epoch): 308s + [Epoch 289][60/66] loss=0.828196 avg=0.866724 VRAM=39.2GiB | 96.6% done | ETA(epoch): 115s + Train loss: 0.864544 (1269.3s) ETA: 234min + Val loss: 0.910646 [t_0.0-0.2=1.0690 t_0.2-0.4=1.0234 t_0.4-0.6=0.8341 t_0.6-0.8=0.7443 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 290/299 (97% done) --- + [Epoch 290][10/66] loss=0.858155 avg=0.876173 VRAM=39.5GiB | 96.7% done | ETA(epoch): 1077s + [Epoch 290][20/66] loss=0.719533 avg=0.858706 VRAM=39.2GiB | 96.8% done | ETA(epoch): 884s + [Epoch 290][30/66] loss=0.936563 avg=0.862165 VRAM=39.5GiB | 96.8% done | ETA(epoch): 692s + [Epoch 290][40/66] loss=0.852258 avg=0.865450 VRAM=39.2GiB | 96.9% done | ETA(epoch): 500s + [Epoch 290][50/66] loss=0.869154 avg=0.869534 VRAM=39.5GiB | 96.9% done | ETA(epoch): 308s + [Epoch 290][60/66] loss=0.839787 avg=0.861828 VRAM=39.2GiB | 97.0% done | ETA(epoch): 115s + Train loss: 0.864741 (1269.5s) ETA: 210min + Val loss: 0.828590 [t_0.0-0.2=1.0568 t_0.2-0.4=0.9631 t_0.4-0.6=0.8720 t_0.6-0.8=0.7220 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 22.7/188.5 GiB (12.0%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 291/299 (97% done) --- + [Epoch 291][10/66] loss=0.759461 avg=0.823194 VRAM=39.5GiB | 97.1% done | ETA(epoch): 1099s + [Epoch 291][20/66] loss=0.822337 avg=0.839936 VRAM=39.2GiB | 97.1% done | ETA(epoch): 894s + [Epoch 291][30/66] loss=0.853179 avg=0.857961 VRAM=39.5GiB | 97.2% done | ETA(epoch): 697s + [Epoch 291][40/66] loss=0.758673 avg=0.854912 VRAM=39.2GiB | 97.2% done | ETA(epoch): 503s + [Epoch 291][50/66] loss=0.857769 avg=0.851582 VRAM=39.5GiB | 97.3% done | ETA(epoch): 309s + [Epoch 291][60/66] loss=1.021596 avg=0.854507 VRAM=39.2GiB | 97.3% done | ETA(epoch): 116s + Train loss: 0.856769 (1274.2s) ETA: 187min + Val loss: 0.869638 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9669 t_0.4-0.6=0.8422 t_0.6-0.8=0.7168 t_0.8-1.0=0.6510] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 292/299 (97% done) --- + [Epoch 292][10/66] loss=0.918137 avg=0.856004 VRAM=39.5GiB | 97.4% done | ETA(epoch): 1075s + [Epoch 292][20/66] loss=0.820423 avg=0.875833 VRAM=39.2GiB | 97.4% done | ETA(epoch): 884s + [Epoch 292][30/66] loss=0.956374 avg=0.877617 VRAM=39.5GiB | 97.5% done | ETA(epoch): 692s + [Epoch 292][40/66] loss=0.926248 avg=0.867839 VRAM=39.2GiB | 97.5% done | ETA(epoch): 500s + [Epoch 292][50/66] loss=0.896582 avg=0.862168 VRAM=39.5GiB | 97.6% done | ETA(epoch): 308s + [Epoch 292][60/66] loss=0.790935 avg=0.855561 VRAM=39.2GiB | 97.6% done | ETA(epoch): 115s + Train loss: 0.849939 (1270.0s) ETA: 163min + Val loss: 0.885457 [t_0.0-0.2=1.0809 t_0.2-0.4=1.0065 t_0.4-0.6=0.8512 t_0.6-0.8=0.7174 t_0.8-1.0=0.6359] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 293/299 (98% done) --- + [Epoch 293][10/66] loss=0.806415 avg=0.862020 VRAM=39.5GiB | 97.7% done | ETA(epoch): 1076s + [Epoch 293][20/66] loss=0.812157 avg=0.854610 VRAM=39.2GiB | 97.8% done | ETA(epoch): 884s + [Epoch 293][30/66] loss=0.855327 avg=0.852369 VRAM=39.5GiB | 97.8% done | ETA(epoch): 692s + [Epoch 293][40/66] loss=0.914998 avg=0.859803 VRAM=39.2GiB | 97.9% done | ETA(epoch): 500s + [Epoch 293][50/66] loss=0.826320 avg=0.858213 VRAM=39.5GiB | 97.9% done | ETA(epoch): 308s + [Epoch 293][60/66] loss=0.977803 avg=0.856960 VRAM=39.2GiB | 98.0% done | ETA(epoch): 115s + Train loss: 0.856030 (1269.0s) ETA: 140min + Val loss: 0.860125 [t_0.0-0.2=1.0565 t_0.2-0.4=1.0098 t_0.4-0.6=0.8479 t_0.6-0.8=0.6864 t_0.8-1.0=0.6626] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 294/299 (98% done) --- + [Epoch 294][10/66] loss=0.867828 avg=0.878410 VRAM=39.5GiB | 98.1% done | ETA(epoch): 1076s + [Epoch 294][20/66] loss=0.805575 avg=0.835481 VRAM=39.2GiB | 98.1% done | ETA(epoch): 885s + [Epoch 294][30/66] loss=0.840907 avg=0.829142 VRAM=39.5GiB | 98.2% done | ETA(epoch): 692s + [Epoch 294][40/66] loss=0.928400 avg=0.840025 VRAM=39.2GiB | 98.2% done | ETA(epoch): 500s + [Epoch 294][50/66] loss=0.858501 avg=0.843049 VRAM=39.5GiB | 98.3% done | ETA(epoch): 308s + [Epoch 294][60/66] loss=0.823769 avg=0.849271 VRAM=39.2GiB | 98.3% done | ETA(epoch): 115s + Train loss: 0.853474 (1269.9s) ETA: 117min + Val loss: 0.845561 [t_0.0-0.2=1.0702 t_0.2-0.4=1.0234 t_0.4-0.6=0.8462 t_0.6-0.8=0.7096 t_0.8-1.0=0.6522] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 295/299 (98% done) --- + [Epoch 295][10/66] loss=0.927946 avg=0.871862 VRAM=39.5GiB | 98.4% done | ETA(epoch): 1078s + [Epoch 295][20/66] loss=0.813436 avg=0.868485 VRAM=39.2GiB | 98.4% done | ETA(epoch): 886s + [Epoch 295][30/66] loss=0.905602 avg=0.876589 VRAM=39.5GiB | 98.5% done | ETA(epoch): 695s + [Epoch 295][40/66] loss=0.803431 avg=0.865105 VRAM=39.2GiB | 98.5% done | ETA(epoch): 502s + [Epoch 295][50/66] loss=0.774241 avg=0.855374 VRAM=39.5GiB | 98.6% done | ETA(epoch): 309s + [Epoch 295][60/66] loss=0.954709 avg=0.855321 VRAM=39.2GiB | 98.6% done | ETA(epoch): 116s + Train loss: 0.853116 (1272.7s) ETA: 93min + Val loss: 0.861479 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0051 t_0.4-0.6=0.8099 t_0.6-0.8=0.7068 t_0.8-1.0=0.6576] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 296/299 (99% done) --- + [Epoch 296][10/66] loss=0.840434 avg=0.854529 VRAM=39.5GiB | 98.7% done | ETA(epoch): 1075s + [Epoch 296][20/66] loss=0.822980 avg=0.848409 VRAM=39.2GiB | 98.8% done | ETA(epoch): 884s + [Epoch 296][30/66] loss=0.679618 avg=0.845976 VRAM=39.5GiB | 98.8% done | ETA(epoch): 692s + [Epoch 296][40/66] loss=0.834561 avg=0.848796 VRAM=39.2GiB | 98.9% done | ETA(epoch): 500s + [Epoch 296][50/66] loss=0.892008 avg=0.842734 VRAM=39.5GiB | 98.9% done | ETA(epoch): 308s + [Epoch 296][60/66] loss=0.829310 avg=0.840741 VRAM=39.2GiB | 99.0% done | ETA(epoch): 115s + Train loss: 0.842916 (1269.7s) ETA: 70min + Val loss: 0.874096 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9796 t_0.4-0.6=0.8181 t_0.6-0.8=0.7176 t_0.8-1.0=0.6608] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 22.3/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 297/299 (99% done) --- + [Epoch 297][10/66] loss=0.821086 avg=0.856479 VRAM=39.5GiB | 99.1% done | ETA(epoch): 1078s + [Epoch 297][20/66] loss=0.956388 avg=0.855623 VRAM=39.2GiB | 99.1% done | ETA(epoch): 895s + [Epoch 297][30/66] loss=0.876040 avg=0.855998 VRAM=39.5GiB | 99.2% done | ETA(epoch): 698s + [Epoch 297][40/66] loss=0.962574 avg=0.870346 VRAM=39.2GiB | 99.2% done | ETA(epoch): 503s + [Epoch 297][50/66] loss=0.802480 avg=0.865274 VRAM=39.5GiB | 99.3% done | ETA(epoch): 309s + [Epoch 297][60/66] loss=0.910297 avg=0.870867 VRAM=39.2GiB | 99.3% done | ETA(epoch): 116s + Train loss: 0.873906 (1274.4s) ETA: 47min + Val loss: 0.861062 [t_0.0-0.2=1.0797 t_0.2-0.4=1.0044 t_0.4-0.6=0.8333 t_0.6-0.8=0.7118 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 298/299 (99% done) --- + [Epoch 298][10/66] loss=0.747184 avg=0.819028 VRAM=39.5GiB | 99.4% done | ETA(epoch): 1091s + [Epoch 298][20/66] loss=0.763173 avg=0.839995 VRAM=39.2GiB | 99.4% done | ETA(epoch): 891s + [Epoch 298][30/66] loss=0.926816 avg=0.852233 VRAM=39.5GiB | 99.5% done | ETA(epoch): 696s + [Epoch 298][40/66] loss=0.827648 avg=0.858143 VRAM=39.2GiB | 99.5% done | ETA(epoch): 502s + [Epoch 298][50/66] loss=0.895967 avg=0.861940 VRAM=39.5GiB | 99.6% done | ETA(epoch): 309s + [Epoch 298][60/66] loss=0.936083 avg=0.860988 VRAM=39.2GiB | 99.6% done | ETA(epoch): 116s + Train loss: 0.860597 (1272.7s) ETA: 23min + Val loss: 0.893848 [t_0.0-0.2=1.0825 t_0.2-0.4=0.9892 t_0.4-0.6=0.8550 t_0.6-0.8=0.7166 t_0.8-1.0=0.6563] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.8/93.1 GiB (41.7%) + +--- Epoch 299/299 (100% done) --- + [Epoch 299][10/66] loss=0.867294 avg=0.857136 VRAM=39.5GiB | 99.7% done | ETA(epoch): 1077s + [Epoch 299][20/66] loss=0.867343 avg=0.857795 VRAM=39.2GiB | 99.8% done | ETA(epoch): 885s + [Epoch 299][30/66] loss=0.854307 avg=0.851030 VRAM=39.5GiB | 99.8% done | ETA(epoch): 692s + [Epoch 299][40/66] loss=0.867914 avg=0.857910 VRAM=39.2GiB | 99.9% done | ETA(epoch): 500s + [Epoch 299][50/66] loss=0.834538 avg=0.860972 VRAM=39.5GiB | 99.9% done | ETA(epoch): 308s + [Epoch 299][60/66] loss=0.906946 avg=0.859722 VRAM=39.2GiB | 100.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 19799 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 19799 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 19799 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 19799 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 19799 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 19799 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 19799 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 19799 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 19799 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 19799 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_768_H100_b/milestone_vis/milestone_grid_test_5.png + Train loss: 0.859830 (2308.1s) ETA: 0min + Val loss: 0.868494 [t_0.0-0.2=1.0629 t_0.2-0.4=0.9651 t_0.4-0.6=0.8547 t_0.6-0.8=0.7299 t_0.8-1.0=0.6515] + Checkpoint saved: output/lora_512_dim_768_H100_b/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 22.8/188.5 GiB (12.1%) | VRAM: 38.8/93.1 GiB (41.7%) + [LossCurve WARN] could not save loss curve: No module named 'matplotlib' + +====================================================================== +Training complete. +====================================================================== +wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: uploading config.yaml 3.5KB/3.5KB (0.5s) + m wandb: uploading config.yaml 3.5KB/3.5KB (0.5s) + m wandb: uploading config.yaml 3.5KB/3.5KB (0.5s) + m wandb: uploading config.yaml 3.5KB/3.5KB (0.5s) + m wandb: +wandb: Run summary: +wandb: adapter_lr 1e-05 +wandb: best_val_loss 0.8193 +wandb: cpu_pct 0.7 +wandb: epoch 299 +wandb: final_train_loss 0.85983 +wandb: flops/batch_size 1 +wandb: flops/hdc2a 362103701504 +wandb: flops/total 274465591212676 +wandb: flops/transformer 274103487511172 +wandb: global_step 19793 +wandb: +10 ... +wandb: +wandb: View run lora_512_dim_768_H100_b-train-20260505-152359 at: https://wandb.ai/hku-xg-boost/lora_512_dim_768_H100_b/runs/s8w3h8ai +wandb: View project at: https://wandb.ai/hku-xg-boost/lora_512_dim_768_H100_b +wandb: Synced 4 W&B file(s), 110 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260505_152359-s8w3h8ai/logs + +====================================================================== +Command: train_script_ablation_b.py --name lora_512_dim_768_H100_b --batch-size 6 --adapter-lr 1e-3 --num-epochs 300 --lora-lr 1e-4 --adapter-mlp --lora-rank 512 --fusion-dim 768 --seed 42 --resume output/lora_512_dim_768_H100_b/checkpoint_epoch_0299 +====================================================================== +Run dir : output/lora_512_dim_768_H100_b +Log file: output/lora_512_dim_768_H100_b/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run km3o1lau (0.3s) + m wandb: setting up run km3o1lau (0.3s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260512_102853-km3o1lau +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_512_dim_768_H100_b-train-20260512-102853 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_512_dim_768_H100_b +wandb: View run at https://wandb.ai/hku-xg-boost/lora_512_dim_768_H100_b/runs/km3o1lau + +Run Summary: + name lora_512_dim_768_H100_b + fusion MLP (lightweight) + LoRA ON (rank=512) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_512_dim_768_H100_b + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 300 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_512_dim_768_H100_b + Resume: + resume_from output/lora_512_dim_768_H100_b/checkpoint_epoch_0299 +[MEM @ pre-flight] RAM: 12.8/188.5 GiB (6.8%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.6s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 13.2/188.5 GiB (7.0%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (67.3s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 15.7/188.5 GiB (8.3%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating Ablation Adapter (b: seg-only) +============================================================ + Ablation mode: (b) Segmentation-only condition +HDC²A: 4.3M params +Control: 0.0M params +Total trainable: 4.3M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=512, alpha=512.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 157.29M + +Parameter Statistics: + HDC²A Adapter: total=4.3M trainable=4.3M + ControlNet (frozen): total=4290.8M LoRA trainable=157.29M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 4.3M + LoRA 157.29M = 161.63M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 20 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~4800 steps + +============================================================ + [6/8] Resuming from Checkpoint +============================================================ + Path: output/lora_512_dim_768_H100_b/checkpoint_epoch_0299 + Loaded control_params.pt: 126 keys, 534 missing from ckpt + Restored LR scheduler state from scheduler.pt + Loaded checkpoint from epoch 299, loss=0.868494 + best_val_loss=0.819299, global_step=5100 + Resuming from epoch 300, global_step=5100, best_val_loss=0.819299 + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0170, std=0.5117 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.332179 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.43 GiB + [test 4/4] Gradient flow check... + HDC²A: 20/20 params have non-zero grad + Control: 50/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.3.weight: 0.057861 + semantic_encoder.conv_stem.6.weight: 0.054688 + semantic_encoder.conv_stem.0.weight: 0.049805 + output_scale: 0.036865 + semantic_encoder.proj.weight: 0.028442 + Test result: PASSED +[MEM @ after test] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.8/93.1 GiB (41.7%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 362.104 GFLOPs + transformer: 274103.488 GFLOPs + total: 274465.591 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 300 epochs × 66 steps = 19800 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 1980, 3960, 5940, 7920, 9900, 11879, 13859, 15839, 17819, 19799] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +====================================================================== +Training complete. +====================================================================== +Traceback (most recent call last): + File "/home/xg_wang_group/SynthUrbanSAT/train_script_ablation_b.py", line 1396, in + main() + File "/home/xg_wang_group/SynthUrbanSAT/train_script_ablation_b.py", line 1387, in main + "final_train_loss": train_loss, + ^^^^^^^^^^ +UnboundLocalError: cannot access local variable 'train_loss' where it is not associated with a value diff --git a/lora_baseline_h100/tmux_run.log b/lora_baseline_h100/tmux_run.log new file mode 100644 index 0000000000000000000000000000000000000000..1a76062eceb0d0325da499ce7dbc673c8ebee3d0 --- /dev/null +++ b/lora_baseline_h100/tmux_run.log @@ -0,0 +1,108 @@ +Run dir : output/lora_baseline_h100 +Log file: output/lora_baseline_h100/train.log +GPU: NVIDIA GeForce RTX 5090 | VRAM: 31.4 GiB | PyTorch: 2.11.0+cu130 + +WARNING: WandB init failed — training will continue without logging. + Error: No API key configured. Use `wandb login` to log in. + Fix: check WANDB_API_KEY permissions, or run with --no-wandb to silence this. + +wandb: WARNING Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to https://wandb.me/wandb-init. + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_baseline_h100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 4 + learning_rate 0.0006 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project lora_baseline_h100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.6/188.5 GiB (6.2%) | VRAM: 0.0/31.4 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA GeForce RTX 5090 (31.4 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.6s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 11.9/188.5 GiB (6.3%) | VRAM: 0.2/31.4 GiB (0.5%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... +Traceback (most recent call last): + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1305, in + main() + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 771, in main + transformer = load_transformer( + ^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/utility.py", line 143, in load_transformer + dequant_sd, fp8_count = dequant_fp8_state_dict(transformer_path, device=device, dtype=dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/utility.py", line 85, in dequant_fp8_state_dict + dequant_sd[key] = (tensor.to(torch.float32) * scale).to(dtype).to(device) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 288.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 102.56 MiB is free. Including non-PyTorch memory, this process has 31.24 GiB memory in use. Of the allocated memory 30.65 GiB is allocated by PyTorch, and 8.62 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf) diff --git a/lora_baseline_h100/train.log b/lora_baseline_h100/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4b94c6cf8bca37a40dfb1efe8b58adf635fb4436 --- /dev/null +++ b/lora_baseline_h100/train.log @@ -0,0 +1,1022 @@ +Run dir : output/lora_baseline_h100 +Log file: output/lora_baseline_h100/train.log +GPU: NVIDIA GeForce RTX 5090 | VRAM: 31.4 GiB | PyTorch: 2.11.0+cu130 + +WARNING: WandB init failed — training will continue without logging. + Error: No API key configured. Use `wandb login` to log in. + Fix: check WANDB_API_KEY permissions, or run with --no-wandb to silence this. + +wandb: WARNING Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to https://wandb.me/wandb-init. + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_baseline_h100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 4 + learning_rate 0.0006 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project lora_baseline_h100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 10.4/188.5 GiB (5.5%) | VRAM: 0.0/31.4 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA GeForce RTX 5090 (31.4 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.5s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 10.7/188.5 GiB (5.7%) | VRAM: 0.2/31.4 GiB (0.5%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... +Traceback (most recent call last): + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1305, in + main() + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 771, in main + transformer = load_transformer( + ^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/utility.py", line 143, in load_transformer + dequant_sd, fp8_count = dequant_fp8_state_dict(transformer_path, device=device, dtype=dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/utility.py", line 85, in dequant_fp8_state_dict + dequant_sd[key] = (tensor.to(torch.float32) * scale).to(dtype).to(device) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 288.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 102.56 MiB is free. Including non-PyTorch memory, this process has 31.24 GiB memory in use. Of the allocated memory 30.65 GiB is allocated by PyTorch, and 8.62 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf) +Run dir : output/lora_baseline_h100 +Log file: output/lora_baseline_h100/train.log +GPU: NVIDIA GeForce RTX 5090 | VRAM: 31.4 GiB | PyTorch: 2.11.0+cu130 + +WARNING: WandB init failed — training will continue without logging. + Error: No API key configured. Use `wandb login` to log in. + Fix: check WANDB_API_KEY permissions, or run with --no-wandb to silence this. + +wandb: WARNING Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to https://wandb.me/wandb-init. + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_baseline_h100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 4 + learning_rate 0.0006 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project lora_baseline_h100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.6/188.5 GiB (6.2%) | VRAM: 0.0/31.4 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA GeForce RTX 5090 (31.4 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.6s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 11.9/188.5 GiB (6.3%) | VRAM: 0.2/31.4 GiB (0.5%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... +Traceback (most recent call last): + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1305, in + main() + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 771, in main + transformer = load_transformer( + ^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/utility.py", line 143, in load_transformer + dequant_sd, fp8_count = dequant_fp8_state_dict(transformer_path, device=device, dtype=dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/utility.py", line 85, in dequant_fp8_state_dict + dequant_sd[key] = (tensor.to(torch.float32) * scale).to(dtype).to(device) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 288.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 102.56 MiB is free. Including non-PyTorch memory, this process has 31.24 GiB memory in use. Of the allocated memory 30.65 GiB is allocated by PyTorch, and 8.62 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf) +Run dir : output/lora_baseline_h100 +Log file: output/lora_baseline_h100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: (1) Create a W&B account +wandb: (2) Use an existing W&B account +wandb: (3) Don't visualize my results +wandb: Enter your choice: wandb: You chose 'Create a W&B account' +wandb: Create an account here: https://wandb.ai/authorize?signup=true&ref=models +wandb: After creating your account, create a new API key and store it securely. +wandb: Paste your API key and hit enter:wandb: No netrc file found, creating one. +wandb: Appending key for api.wandb.ai to your netrc file: /home/xg_wang_group/.netrc +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run hu0pyawd (0.3s) + m wandb: setting up run hu0pyawd (0.3s) + m wandb: setting up run hu0pyawd (0.3s) + m wandb: setting up run hu0pyawd (0.3s) + m wandb: setting up run hu0pyawd (0.3s) + m wandb: setting up run hu0pyawd (0.8s) + m wandb: setting up run hu0pyawd (0.8s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260416_114715-hu0pyawd +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_baseline_h100-train-20260416-114534 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_baseline_h100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_baseline_h100/runs/hu0pyawd + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_baseline_h100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 4 + learning_rate 0.0006 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project lora_baseline_h100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 10.1/188.5 GiB (5.3%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.5s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 10.4/188.5 GiB (5.5%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (57.1s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 11.9/188.5 GiB (6.3%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ +HDC²A: 52.4M params +Control: 0.0M params +Total trainable: 52.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=32, alpha=32.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 9.83M + +Parameter Statistics: + HDC²A Adapter: total=52.4M trainable=52.4M + ControlNet (frozen): total=4143.4M LoRA trainable=9.83M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 52.4M + LoRA 9.83M = 62.19M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=3.00e-04, backbone_lr=0.00e+00 + param_group 'adapter': 112 tensors, lr=3.00e-04 + Scheduler: 400 warmup steps → cosine over ~12500 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0413, std=0.5156 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.335436 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 49.21 GiB + [test 4/4] Gradient flow check... + HDC²A: 112/112 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.002731 + depth_encoder.conv_stem.6.weight: 0.002289 + W_s.weight: 0.002197 + W_d.weight: 0.002029 + fusion_blocks.0.ffn_sem.2.weight: 0.001831 + Test result: PASSED +[MEM @ after test] RAM: 12.5/188.5 GiB (6.6%) | VRAM: 38.1/93.1 GiB (40.9%) + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=4 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=4 +[Data] Test: 30 samples, batch_size=4 + +====================================================================== +Starting training: 500 epochs × 100 steps = 50000 total steps + batch_size=4, grad_accum=4, world_size=1, effective_bs=16 + adapter_lr=3.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 5000, 10000, 15000, 20000, 25000, 29999, 34999, 39999, 44999, 49999] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/499 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_baseline_h100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_baseline_h100/milestone_vis/milestone_grid_train_1.png +Traceback (most recent call last): + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1305, in + main() + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1166, in main + train_loss, _epoch_opt_steps = train_one_epoch( + ^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/train.py", line 250, in train_one_epoch + step_vis_callback(global_step, batch) + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1147, in _milestone_step_callback + _run_milestone_vis(_global_step) + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1086, in _run_milestone_vis + gen_rgb = generate_overfit_samples( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/overfit.py", line 593, in generate_overfit_samples + control_context = hdc2a(seg, depth) + ^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/models.py", line 252, in forward + T_s, T_d = block(T_s, T_d) + ^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/models.py", line 179, in forward + k_s = self.rope(k_s) + ^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/models.py", line 51, in forward + return (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +KeyboardInterrupt +Exception ignored in atexit callback: .teardown_atexit at 0x6ffd34c7fce0> +Traceback (most recent call last): + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/wandb/sdk/lib/service/service_connection.py", line 73, in teardown_atexit + conn.teardown(hooks.exit_code) + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/wandb/sdk/lib/service/service_connection.py", line 355, in teardown + return self._proc.join() + ^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/wandb/sdk/lib/service/service_process.py", line 103, in join + return self._process.wait() + ^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/subprocess.py", line 2053, in _wait + (pid, sts) = self._try_wait(0) + ^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/subprocess.py", line 2011, in _try_wait + (pid, sts) = os.waitpid(self.pid, wait_flags) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +KeyboardInterrupt: +Run dir : output/lora_baseline_h100 +Log file: output/lora_baseline_h100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from /home/xg_wang_group/.netrc. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run 25w3do53 (0.2s) + m wandb: setting up run 25w3do53 (0.2s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260416_115235-25w3do53 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_baseline_h100-train-20260416-115235 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_baseline_h100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_baseline_h100/runs/25w3do53 + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_baseline_h100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 4 + learning_rate 0.0006 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project lora_baseline_h100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 10.6/188.5 GiB (5.6%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.5s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 10.9/188.5 GiB (5.8%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (54.9s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 12.2/188.5 GiB (6.5%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ +HDC²A: 52.4M params +Control: 0.0M params +Total trainable: 52.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=32, alpha=32.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 9.83M + +Parameter Statistics: + HDC²A Adapter: total=52.4M trainable=52.4M + ControlNet (frozen): total=4143.4M LoRA trainable=9.83M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 52.4M + LoRA 9.83M = 62.19M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=3.00e-04, backbone_lr=0.00e+00 + param_group 'adapter': 112 tensors, lr=3.00e-04 + Scheduler: 400 warmup steps → cosine over ~12500 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0413, std=0.5156 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.335436 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 49.21 GiB + [test 4/4] Gradient flow check... + HDC²A: 112/112 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.002731 + depth_encoder.conv_stem.6.weight: 0.002289 + W_s.weight: 0.002182 + W_d.weight: 0.002029 + fusion_blocks.0.ffn_sem.2.weight: 0.001831 + Test result: PASSED +[MEM @ after test] RAM: 12.7/188.5 GiB (6.8%) | VRAM: 38.1/93.1 GiB (40.9%) + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=4 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=4 +[Data] Test: 30 samples, batch_size=4 + +====================================================================== +Starting training: 500 epochs × 100 steps = 50000 total steps + batch_size=4, grad_accum=4, world_size=1, effective_bs=16 + adapter_lr=3.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 5000, 10000, 15000, 20000, 25000, 29999, 34999, 39999, 44999, 49999] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/499 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_baseline_h100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_baseline_h100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_baseline_h100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_baseline_h100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_baseline_h100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_baseline_h100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_baseline_h100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_baseline_h100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_baseline_h100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_baseline_h100/milestone_vis/milestone_grid_test_5.png + [HF Upload WARN] upload failed: No module named 'ControlNet_training' + [Epoch 0][10/100] loss=0.941065 avg=0.924597 VRAM=38.6GiB | 0.0% done | ETA(epoch): 10631s + [Epoch 0][20/100] loss=0.800769 avg=0.912605 VRAM=38.5GiB | 0.0% done | ETA(epoch): 5255s + [Epoch 0][30/100] loss=0.999155 avg=0.911449 VRAM=38.6GiB | 0.1% done | ETA(epoch): 3384s + [Epoch 0][40/100] loss=0.830989 avg=0.906137 VRAM=38.5GiB | 0.1% done | ETA(epoch): 2374s + [Epoch 0][50/100] loss=0.775814 avg=0.907010 VRAM=38.6GiB | 0.1% done | ETA(epoch): 1716s + [Epoch 0][60/100] loss=0.878590 avg=0.908445 VRAM=38.5GiB | 0.1% done | ETA(epoch): 1232s + [Epoch 0][70/100] loss=1.031628 avg=0.911537 VRAM=38.6GiB | 0.1% done | ETA(epoch): 849s + [Epoch 0][80/100] loss=0.866052 avg=0.914041 VRAM=38.5GiB | 0.2% done | ETA(epoch): 528s + [Epoch 0][90/100] loss=0.915645 avg=0.913495 VRAM=38.6GiB | 0.2% done | ETA(epoch): 250s + [Epoch 0][100/100] loss=0.869309 avg=0.913075 VRAM=38.5GiB | 0.2% done | ETA(epoch): 0s + Train loss: 0.913075 (2379.3s) ETA: 19824min + Val loss: 0.938541 [t_0.0-0.2=1.1016 t_0.2-0.4=1.0422 t_0.4-0.6=0.8825 t_0.6-0.8=0.7715 t_0.8-1.0=0.9322] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 1/499 (0% done) --- + [Epoch 1][10/100] loss=0.809276 avg=0.905533 VRAM=38.6GiB | 0.2% done | ETA(epoch): 1193s + [Epoch 1][20/100] loss=0.830221 avg=0.909845 VRAM=38.5GiB | 0.2% done | ETA(epoch): 1062s + [Epoch 1][30/100] loss=0.944313 avg=0.909747 VRAM=38.6GiB | 0.3% done | ETA(epoch): 929s + [Epoch 1][40/100] loss=0.910843 avg=0.912359 VRAM=38.5GiB | 0.3% done | ETA(epoch): 796s + [Epoch 1][50/100] loss=0.963525 avg=0.911716 VRAM=38.6GiB | 0.3% done | ETA(epoch): 664s + [Epoch 1][60/100] loss=0.984976 avg=0.909685 VRAM=38.5GiB | 0.3% done | ETA(epoch): 531s + [Epoch 1][70/100] loss=0.803163 avg=0.910757 VRAM=38.6GiB | 0.3% done | ETA(epoch): 398s + [Epoch 1][80/100] loss=0.954126 avg=0.915441 VRAM=38.5GiB | 0.4% done | ETA(epoch): 266s + [Epoch 1][90/100] loss=0.985108 avg=0.914229 VRAM=38.6GiB | 0.4% done | ETA(epoch): 133s + [Epoch 1][100/100] loss=1.071944 avg=0.915659 VRAM=38.5GiB | 0.4% done | ETA(epoch): 0s + Train loss: 0.915659 (1327.6s) ETA: 15810min + Val loss: 0.930462 [t_0.0-0.2=1.0887 t_0.2-0.4=1.0227 t_0.4-0.6=0.8967 t_0.6-0.8=0.7742 t_0.8-1.0=0.9710] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 15.5/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 2/499 (0% done) --- + [Epoch 2][10/100] loss=0.896954 avg=0.922830 VRAM=38.6GiB | 0.4% done | ETA(epoch): 1194s + [Epoch 2][20/100] loss=0.879038 avg=0.923363 VRAM=38.5GiB | 0.4% done | ETA(epoch): 1061s + [Epoch 2][30/100] loss=0.824969 avg=0.915989 VRAM=38.6GiB | 0.5% done | ETA(epoch): 929s + [Epoch 2][40/100] loss=0.882990 avg=0.917364 VRAM=38.5GiB | 0.5% done | ETA(epoch): 796s + [Epoch 2][50/100] loss=1.054123 avg=0.913184 VRAM=38.6GiB | 0.5% done | ETA(epoch): 663s + [Epoch 2][60/100] loss=0.863263 avg=0.912003 VRAM=38.5GiB | 0.5% done | ETA(epoch): 531s + [Epoch 2][70/100] loss=0.982849 avg=0.912602 VRAM=38.6GiB | 0.5% done | ETA(epoch): 398s + [Epoch 2][80/100] loss=0.842028 avg=0.913022 VRAM=38.5GiB | 0.6% done | ETA(epoch): 265s + [Epoch 2][90/100] loss=0.888870 avg=0.917269 VRAM=38.6GiB | 0.6% done | ETA(epoch): 133s + [Epoch 2][100/100] loss=0.917875 avg=0.915649 VRAM=38.5GiB | 0.6% done | ETA(epoch): 0s + Train loss: 0.915649 (1330.5s) ETA: 14464min + Val loss: 0.932581 [t_0.0-0.2=1.1004 t_0.2-0.4=1.0259 t_0.4-0.6=0.8721 t_0.6-0.8=0.7633 t_0.8-1.0=0.9520] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 15.5/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 3/499 (1% done) --- + [Epoch 3][10/100] loss=0.798546 avg=0.892938 VRAM=38.6GiB | 0.6% done | ETA(epoch): 1192s + [Epoch 3][20/100] loss=0.986750 avg=0.928209 VRAM=38.5GiB | 0.6% done | ETA(epoch): 1061s + [Epoch 3][30/100] loss=0.774730 avg=0.905527 VRAM=38.6GiB | 0.7% done | ETA(epoch): 928s + [Epoch 3][40/100] loss=1.007182 avg=0.905173 VRAM=38.5GiB | 0.7% done | ETA(epoch): 796s + [Epoch 3][50/100] loss=0.976228 avg=0.912917 VRAM=38.6GiB | 0.7% done | ETA(epoch): 663s + [Epoch 3][60/100] loss=0.981444 avg=0.913284 VRAM=38.5GiB | 0.7% done | ETA(epoch): 530s + [Epoch 3][70/100] loss=0.820385 avg=0.909043 VRAM=38.6GiB | 0.7% done | ETA(epoch): 398s + [Epoch 3][80/100] loss=0.814420 avg=0.905657 VRAM=38.5GiB | 0.8% done | ETA(epoch): 265s + [Epoch 3][90/100] loss=0.834756 avg=0.908909 VRAM=38.6GiB | 0.8% done | ETA(epoch): 133s + [Epoch 3][100/100] loss=1.017020 avg=0.909447 VRAM=38.5GiB | 0.8% done | ETA(epoch): 0s + Train loss: 0.909447 (1326.2s) ETA: 13770min + Val loss: 0.954529 [t_0.0-0.2=1.0967 t_0.2-0.4=1.0011 t_0.4-0.6=0.8903 t_0.6-0.8=0.7404 t_0.8-1.0=0.9429] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 4/499 (1% done) --- + [Epoch 4][10/100] loss=0.863603 avg=0.873102 VRAM=38.6GiB | 0.8% done | ETA(epoch): 1193s + [Epoch 4][20/100] loss=0.797668 avg=0.897814 VRAM=38.5GiB | 0.8% done | ETA(epoch): 1061s + [Epoch 4][30/100] loss=0.956929 avg=0.897195 VRAM=38.6GiB | 0.9% done | ETA(epoch): 929s + [Epoch 4][40/100] loss=0.987086 avg=0.901464 VRAM=38.5GiB | 0.9% done | ETA(epoch): 796s + [Epoch 4][50/100] loss=0.827385 avg=0.904161 VRAM=38.6GiB | 0.9% done | ETA(epoch): 668s + [Epoch 4][60/100] loss=0.785728 avg=0.902909 VRAM=38.5GiB | 0.9% done | ETA(epoch): 534s + [Epoch 4][70/100] loss=0.924116 avg=0.904073 VRAM=38.6GiB | 0.9% done | ETA(epoch): 400s + [Epoch 4][80/100] loss=0.803847 avg=0.898653 VRAM=38.5GiB | 1.0% done | ETA(epoch): 267s + [Epoch 4][90/100] loss=0.906517 avg=0.897017 VRAM=38.6GiB | 1.0% done | ETA(epoch): 133s + [Epoch 4][100/100] loss=0.838926 avg=0.895190 VRAM=38.5GiB | 1.0% done | ETA(epoch): 0s + Train loss: 0.895190 (1332.2s) ETA: 13354min + Val loss: 0.938578 [t_0.0-0.2=1.0855 t_0.2-0.4=1.0255 t_0.4-0.6=0.8686 t_0.6-0.8=0.7556 t_0.8-1.0=0.9351] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0004 +[MEM @ epoch 4 end] RAM: 15.5/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 5/499 (1% done) --- + [Epoch 5][10/100] loss=0.880699 avg=0.898051 VRAM=38.6GiB | 1.0% done | ETA(epoch): 1192s + [Epoch 5][20/100] loss=1.052332 avg=0.889742 VRAM=38.5GiB | 1.0% done | ETA(epoch): 1060s + [Epoch 5][30/100] loss=0.902173 avg=0.890994 VRAM=38.6GiB | 1.1% done | ETA(epoch): 928s + [Epoch 5][40/100] loss=0.933784 avg=0.893495 VRAM=38.5GiB | 1.1% done | ETA(epoch): 801s + [Epoch 5][50/100] loss=0.851236 avg=0.892024 VRAM=38.6GiB | 1.1% done | ETA(epoch): 667s + [Epoch 5][60/100] loss=0.827588 avg=0.896425 VRAM=38.5GiB | 1.1% done | ETA(epoch): 533s + [Epoch 5][70/100] loss=0.910750 avg=0.897489 VRAM=38.6GiB | 1.1% done | ETA(epoch): 399s + [Epoch 5][80/100] loss=0.850334 avg=0.894766 VRAM=38.5GiB | 1.2% done | ETA(epoch): 266s + [Epoch 5][90/100] loss=0.902218 avg=0.895331 VRAM=38.6GiB | 1.2% done | ETA(epoch): 133s + [Epoch 5][100/100] loss=0.984387 avg=0.894165 VRAM=38.5GiB | 1.2% done | ETA(epoch): 0s + Train loss: 0.894165 (1330.1s) ETA: 13066min + Val loss: 0.915973 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0342 t_0.4-0.6=0.8432 t_0.6-0.8=0.7724 t_0.8-1.0=0.9757] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0005 (BEST) + Deleted old checkpoint: checkpoint_epoch_0001 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 15.3/188.5 GiB (8.1%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 6/499 (1% done) --- + [Epoch 6][10/100] loss=0.998760 avg=0.877657 VRAM=38.6GiB | 1.2% done | ETA(epoch): 1194s + [Epoch 6][20/100] loss=0.828451 avg=0.886384 VRAM=38.5GiB | 1.2% done | ETA(epoch): 1063s + [Epoch 6][30/100] loss=0.846599 avg=0.895307 VRAM=38.6GiB | 1.3% done | ETA(epoch): 930s + [Epoch 6][40/100] loss=0.897369 avg=0.900879 VRAM=38.5GiB | 1.3% done | ETA(epoch): 797s + [Epoch 6][50/100] loss=0.937362 avg=0.898990 VRAM=38.6GiB | 1.3% done | ETA(epoch): 664s + [Epoch 6][60/100] loss=0.871279 avg=0.899144 VRAM=38.5GiB | 1.3% done | ETA(epoch): 531s + [Epoch 6][70/100] loss=0.882198 avg=0.899551 VRAM=38.6GiB | 1.3% done | ETA(epoch): 398s + [Epoch 6][80/100] loss=0.924926 avg=0.897149 VRAM=38.5GiB | 1.4% done | ETA(epoch): 265s + [Epoch 6][90/100] loss=0.879220 avg=0.898957 VRAM=38.6GiB | 1.4% done | ETA(epoch): 133s + [Epoch 6][100/100] loss=0.926145 avg=0.902229 VRAM=38.5GiB | 1.4% done | ETA(epoch): 0s + Train loss: 0.902229 (1327.3s) ETA: 12851min + Val loss: 0.919442 [t_0.0-0.2=1.1017 t_0.2-0.4=1.0331 t_0.4-0.6=0.8386 t_0.6-0.8=0.7633 t_0.8-1.0=0.8907] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0006 + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 15.6/188.5 GiB (8.3%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 7/499 (1% done) --- + [Epoch 7][10/100] loss=0.861784 avg=0.881602 VRAM=38.6GiB | 1.4% done | ETA(epoch): 1192s + [Epoch 7][20/100] loss=0.919683 avg=0.878720 VRAM=38.5GiB | 1.4% done | ETA(epoch): 1061s + [Epoch 7][30/100] loss=0.962979 avg=0.882661 VRAM=38.6GiB | 1.5% done | ETA(epoch): 928s + [Epoch 7][40/100] loss=0.930568 avg=0.884640 VRAM=38.5GiB | 1.5% done | ETA(epoch): 796s + [Epoch 7][50/100] loss=0.974388 avg=0.884026 VRAM=38.6GiB | 1.5% done | ETA(epoch): 663s + [Epoch 7][60/100] loss=0.899950 avg=0.882088 VRAM=38.5GiB | 1.5% done | ETA(epoch): 530s + [Epoch 7][70/100] loss=0.949109 avg=0.885325 VRAM=38.6GiB | 1.5% done | ETA(epoch): 398s + [Epoch 7][80/100] loss=0.953958 avg=0.885340 VRAM=38.5GiB | 1.6% done | ETA(epoch): 265s + [Epoch 7][90/100] loss=0.912331 avg=0.888319 VRAM=38.6GiB | 1.6% done | ETA(epoch): 133s + [Epoch 7][100/100] loss=0.892251 avg=0.891891 VRAM=38.5GiB | 1.6% done | ETA(epoch): 0s + Train loss: 0.891891 (1326.3s) ETA: 12683min + Val loss: 0.948828 [t_0.0-0.2=1.0931 t_0.2-0.4=1.0054 t_0.4-0.6=0.8854 t_0.6-0.8=0.7767 t_0.8-1.0=0.9222] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 8/499 (2% done) --- + [Epoch 8][10/100] loss=0.793007 avg=0.882597 VRAM=38.6GiB | 1.6% done | ETA(epoch): 1194s + [Epoch 8][20/100] loss=0.778859 avg=0.884551 VRAM=38.5GiB | 1.6% done | ETA(epoch): 1061s + [Epoch 8][30/100] loss=0.949995 avg=0.887610 VRAM=38.6GiB | 1.7% done | ETA(epoch): 928s + [Epoch 8][40/100] loss=0.959300 avg=0.894274 VRAM=38.5GiB | 1.7% done | ETA(epoch): 796s + [Epoch 8][50/100] loss=0.950433 avg=0.897313 VRAM=38.6GiB | 1.7% done | ETA(epoch): 663s + [Epoch 8][60/100] loss=0.888023 avg=0.898194 VRAM=38.5GiB | 1.7% done | ETA(epoch): 531s + [Epoch 8][70/100] loss=0.898501 avg=0.898381 VRAM=38.6GiB | 1.7% done | ETA(epoch): 398s + [Epoch 8][80/100] loss=0.839228 avg=0.900195 VRAM=38.5GiB | 1.8% done | ETA(epoch): 266s + [Epoch 8][90/100] loss=0.945698 avg=0.898329 VRAM=38.6GiB | 1.8% done | ETA(epoch): 133s + [Epoch 8][100/100] loss=0.846462 avg=0.896364 VRAM=38.5GiB | 1.8% done | ETA(epoch): 0s + Train loss: 0.896364 (1328.6s) ETA: 12549min + Val loss: 0.929101 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0113 t_0.4-0.6=0.8559 t_0.6-0.8=0.7663 t_0.8-1.0=0.9477] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0008 +[MEM @ epoch 8 end] RAM: 15.5/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 9/499 (2% done) --- + [Epoch 9][10/100] loss=0.811317 avg=0.850544 VRAM=38.6GiB | 1.8% done | ETA(epoch): 1192s + [Epoch 9][20/100] loss=0.878190 avg=0.864503 VRAM=38.5GiB | 1.8% done | ETA(epoch): 1060s + [Epoch 9][30/100] loss=0.816290 avg=0.875198 VRAM=38.6GiB | 1.9% done | ETA(epoch): 928s + [Epoch 9][40/100] loss=0.899423 avg=0.874578 VRAM=38.5GiB | 1.9% done | ETA(epoch): 795s + [Epoch 9][50/100] loss=0.765660 avg=0.879674 VRAM=38.6GiB | 1.9% done | ETA(epoch): 663s + [Epoch 9][60/100] loss=1.041576 avg=0.888781 VRAM=38.5GiB | 1.9% done | ETA(epoch): 534s + [Epoch 9][70/100] loss=0.967851 avg=0.896216 VRAM=38.6GiB | 1.9% done | ETA(epoch): 400s + [Epoch 9][80/100] loss=0.869709 avg=0.898694 VRAM=38.5GiB | 2.0% done | ETA(epoch): 267s + [Epoch 9][90/100] loss=0.913030 avg=0.894023 VRAM=38.6GiB | 2.0% done | ETA(epoch): 133s + [Epoch 9][100/100] loss=0.905419 avg=0.890292 VRAM=38.5GiB | 2.0% done | ETA(epoch): 0s + Train loss: 0.890292 (1332.6s) ETA: 12440min + Val loss: 0.907056 [t_0.0-0.2=1.0759 t_0.2-0.4=1.0022 t_0.4-0.6=0.8641 t_0.6-0.8=0.7658 t_0.8-1.0=0.8648] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0009 (BEST) + Deleted old checkpoint: checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 10/499 (2% done) --- + [Epoch 10][10/100] loss=0.799796 avg=0.875355 VRAM=38.6GiB | 2.0% done | ETA(epoch): 1193s + [Epoch 10][20/100] loss=0.885095 avg=0.887158 VRAM=38.5GiB | 2.0% done | ETA(epoch): 1061s + [Epoch 10][30/100] loss=0.908264 avg=0.882971 VRAM=38.6GiB | 2.1% done | ETA(epoch): 928s + [Epoch 10][40/100] loss=0.998384 avg=0.885364 VRAM=38.5GiB | 2.1% done | ETA(epoch): 795s + [Epoch 10][50/100] loss=0.967074 avg=0.892462 VRAM=38.6GiB | 2.1% done | ETA(epoch): 665s + [Epoch 10][60/100] loss=0.807719 avg=0.891286 VRAM=38.5GiB | 2.1% done | ETA(epoch): 532s + [Epoch 10][70/100] loss=0.930732 avg=0.894307 VRAM=38.6GiB | 2.1% done | ETA(epoch): 399s + [Epoch 10][80/100] loss=0.934865 avg=0.891560 VRAM=38.5GiB | 2.2% done | ETA(epoch): 266s + [Epoch 10][90/100] loss=0.879800 avg=0.896282 VRAM=38.6GiB | 2.2% done | ETA(epoch): 133s + [Epoch 10][100/100] loss=0.951419 avg=0.893173 VRAM=38.5GiB | 2.2% done | ETA(epoch): 0s + Train loss: 0.893173 (1328.5s) ETA: 12343min + Val loss: 0.915998 [t_0.0-0.2=1.0966 t_0.2-0.4=1.0120 t_0.4-0.6=0.8423 t_0.6-0.8=0.7475 t_0.8-1.0=0.9145] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0010 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 11/499 (2% done) --- + [Epoch 11][10/100] loss=0.898770 avg=0.840691 VRAM=38.6GiB | 2.2% done | ETA(epoch): 1192s + [Epoch 11][20/100] loss=0.902107 avg=0.868226 VRAM=38.5GiB | 2.2% done | ETA(epoch): 1061s + [Epoch 11][30/100] loss=0.899334 avg=0.886091 VRAM=38.6GiB | 2.3% done | ETA(epoch): 929s + [Epoch 11][40/100] loss=0.836345 avg=0.883065 VRAM=38.5GiB | 2.3% done | ETA(epoch): 796s + [Epoch 11][50/100] loss=0.838135 avg=0.884279 VRAM=38.6GiB | 2.3% done | ETA(epoch): 663s + [Epoch 11][60/100] loss=0.886633 avg=0.885590 VRAM=38.5GiB | 2.3% done | ETA(epoch): 531s + [Epoch 11][70/100] loss=0.935280 avg=0.888742 VRAM=38.6GiB | 2.3% done | ETA(epoch): 398s + [Epoch 11][80/100] loss=0.892184 avg=0.887628 VRAM=38.5GiB | 2.4% done | ETA(epoch): 265s + [Epoch 11][90/100] loss=0.885992 avg=0.887280 VRAM=38.6GiB | 2.4% done | ETA(epoch): 133s + [Epoch 11][100/100] loss=0.856227 avg=0.888817 VRAM=38.5GiB | 2.4% done | ETA(epoch): 0s + Train loss: 0.888817 (1326.6s) ETA: 12257min + Val loss: 0.924249 [t_0.0-0.2=1.0761 t_0.2-0.4=1.0630 t_0.4-0.6=0.8336 t_0.6-0.8=0.7874 t_0.8-1.0=0.8219] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 15.5/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 12/499 (2% done) --- + [Epoch 12][10/100] loss=0.879421 avg=0.851119 VRAM=38.6GiB | 2.4% done | ETA(epoch): 1193s + [Epoch 12][20/100] loss=0.906359 avg=0.879505 VRAM=38.5GiB | 2.4% done | ETA(epoch): 1061s + [Epoch 12][30/100] loss=0.807515 avg=0.879843 VRAM=38.6GiB | 2.5% done | ETA(epoch): 929s + [Epoch 12][40/100] loss=0.951137 avg=0.893757 VRAM=38.5GiB | 2.5% done | ETA(epoch): 796s + [Epoch 12][50/100] loss=1.001705 avg=0.896496 VRAM=38.6GiB | 2.5% done | ETA(epoch): 663s + [Epoch 12][60/100] loss=0.866891 avg=0.892637 VRAM=38.5GiB | 2.5% done | ETA(epoch): 530s + [Epoch 12][70/100] loss=0.944366 avg=0.892345 VRAM=38.6GiB | 2.5% done | ETA(epoch): 398s + [Epoch 12][80/100] loss=0.969197 avg=0.891150 VRAM=38.5GiB | 2.6% done | ETA(epoch): 265s + [Epoch 12][90/100] loss=0.948820 avg=0.894898 VRAM=38.6GiB | 2.6% done | ETA(epoch): 133s + [Epoch 12][100/100] loss=0.967688 avg=0.894570 VRAM=38.5GiB | 2.6% done | ETA(epoch): 0s + Train loss: 0.894570 (1325.8s) ETA: 12181min + Val loss: 0.910472 [t_0.0-0.2=1.0759 t_0.2-0.4=0.9960 t_0.4-0.6=0.8730 t_0.6-0.8=0.7371 t_0.8-1.0=0.8862] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0012 +[MEM @ epoch 12 end] RAM: 15.5/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 13/499 (3% done) --- + [Epoch 13][10/100] loss=0.870758 avg=0.880895 VRAM=38.6GiB | 2.6% done | ETA(epoch): 1193s + [Epoch 13][20/100] loss=0.939617 avg=0.881841 VRAM=38.5GiB | 2.6% done | ETA(epoch): 1061s + [Epoch 13][30/100] loss=0.893678 avg=0.891095 VRAM=38.6GiB | 2.7% done | ETA(epoch): 929s + [Epoch 13][40/100] loss=0.894570 avg=0.884680 VRAM=38.5GiB | 2.7% done | ETA(epoch): 796s + [Epoch 13][50/100] loss=0.818397 avg=0.888858 VRAM=38.6GiB | 2.7% done | ETA(epoch): 663s + [Epoch 13][60/100] loss=1.002739 avg=0.890945 VRAM=38.5GiB | 2.7% done | ETA(epoch): 531s + [Epoch 13][70/100] loss=0.937253 avg=0.890811 VRAM=38.6GiB | 2.7% done | ETA(epoch): 398s + [Epoch 13][80/100] loss=0.975271 avg=0.894329 VRAM=38.5GiB | 2.8% done | ETA(epoch): 265s + [Epoch 13][90/100] loss=0.830887 avg=0.896053 VRAM=38.6GiB | 2.8% done | ETA(epoch): 133s + [Epoch 13][100/100] loss=0.878928 avg=0.899050 VRAM=38.5GiB | 2.8% done | ETA(epoch): 0s + Train loss: 0.899050 (1329.5s) ETA: 12113min + Val loss: 0.918244 [t_0.0-0.2=1.0871 t_0.2-0.4=1.0228 t_0.4-0.6=0.8267 t_0.6-0.8=0.7782 t_0.8-1.0=0.8386] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 15.5/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 14/499 (3% done) --- + [Epoch 14][10/100] loss=0.840193 avg=0.838119 VRAM=38.6GiB | 2.8% done | ETA(epoch): 1194s + [Epoch 14][20/100] loss=0.881363 avg=0.867359 VRAM=38.5GiB | 2.8% done | ETA(epoch): 1061s + [Epoch 14][30/100] loss=0.820786 avg=0.866717 VRAM=38.6GiB | 2.9% done | ETA(epoch): 928s + [Epoch 14][40/100] loss=0.826457 avg=0.876714 VRAM=38.5GiB | 2.9% done | ETA(epoch): 795s + [Epoch 14][50/100] loss=0.859007 avg=0.880535 VRAM=38.6GiB | 2.9% done | ETA(epoch): 663s + [Epoch 14][60/100] loss=0.897344 avg=0.887403 VRAM=38.5GiB | 2.9% done | ETA(epoch): 530s + [Epoch 14][70/100] loss=0.832865 avg=0.890516 VRAM=38.6GiB | 2.9% done | ETA(epoch): 399s + [Epoch 14][80/100] loss=0.913491 avg=0.890590 VRAM=38.5GiB | 3.0% done | ETA(epoch): 266s + [Epoch 14][90/100] loss=0.801384 avg=0.887471 VRAM=38.6GiB | 3.0% done | ETA(epoch): 133s + [Epoch 14][100/100] loss=0.826606 avg=0.887343 VRAM=38.5GiB | 3.0% done | ETA(epoch): 0s + Train loss: 0.887343 (1329.0s) ETA: 12052min + Val loss: 0.898047 [t_0.0-0.2=1.0879 t_0.2-0.4=1.0043 t_0.4-0.6=0.8682 t_0.6-0.8=0.7621 t_0.8-1.0=0.8571] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0014 (BEST) + Deleted old checkpoint: checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 15/499 (3% done) --- + [Epoch 15][10/100] loss=0.850908 avg=0.911571 VRAM=38.6GiB | 3.0% done | ETA(epoch): 1192s + [Epoch 15][20/100] loss=0.755047 avg=0.896789 VRAM=38.5GiB | 3.0% done | ETA(epoch): 1060s + [Epoch 15][30/100] loss=0.975014 avg=0.897212 VRAM=38.6GiB | 3.1% done | ETA(epoch): 928s + [Epoch 15][40/100] loss=0.812536 avg=0.906780 VRAM=38.5GiB | 3.1% done | ETA(epoch): 796s + [Epoch 15][50/100] loss=1.010267 avg=0.906544 VRAM=38.6GiB | 3.1% done | ETA(epoch): 666s + [Epoch 15][60/100] loss=0.968819 avg=0.905216 VRAM=38.5GiB | 3.1% done | ETA(epoch): 532s + [Epoch 15][70/100] loss=0.905952 avg=0.905869 VRAM=38.6GiB | 3.1% done | ETA(epoch): 399s + [Epoch 15][80/100] loss=0.867547 avg=0.906110 VRAM=38.5GiB | 3.2% done | ETA(epoch): 266s + [Epoch 15][90/100] loss=0.861936 avg=0.904166 VRAM=38.6GiB | 3.2% done | ETA(epoch): 133s + [Epoch 15][100/100] loss=0.910464 avg=0.902628 VRAM=38.5GiB | 3.2% done | ETA(epoch): 0s + Train loss: 0.902628 (1329.2s) ETA: 11995min + Val loss: 0.952279 [t_0.0-0.2=1.0810 t_0.2-0.4=1.0246 t_0.4-0.6=0.8802 t_0.6-0.8=0.7419 t_0.8-1.0=0.9100] + Checkpoint saved: output/lora_baseline_h100/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 15.4/188.5 GiB (8.2%) | VRAM: 38.3/93.1 GiB (41.1%) + +--- Epoch 16/499 (3% done) --- + [Epoch 16][10/100] loss=0.763465 avg=0.871007 VRAM=38.6GiB | 3.2% done | ETA(epoch): 1193s diff --git a/lora_rank_128_mlp_1k_h100/train.log b/lora_rank_128_mlp_1k_h100/train.log new file mode 100644 index 0000000000000000000000000000000000000000..41773021de0641414f84728f9b15072e15d8001c --- /dev/null +++ b/lora_rank_128_mlp_1k_h100/train.log @@ -0,0 +1,5244 @@ + +====================================================================== +Command: train_script.py --name lora_rank_128_mlp_1k_h100 --batch-size 4 --adapter-lr 1e-3 --lora-lr 1e-4 --adapter-mlp --lora-rank 128 --seed 42 --dataset-dir real_dataset/final_training_dataset_18&19_reduceoverlap_approx1k +====================================================================== +Run dir : output/lora_rank_128_mlp_1k_h100 +Log file: output/lora_rank_128_mlp_1k_h100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + +m +wandb: Waiting for wandb.init()... + +m +wandb: setting up run ztmllvo3 (0.2s) + +m +wandb: setting up run ztmllvo3 (0.2s) + +m +wandb: setting up run ztmllvo3 (0.2s) + +m +wandb: setting up run ztmllvo3 (0.2s) + +m +wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260417_170943-ztmllvo3 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_rank_128_mlp_1k_h100-train-20260417-170943 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_rank_128_mlp_1k_h100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_rank_128_mlp_1k_h100/runs/ztmllvo3 + +Run Summary: + name lora_rank_128_mlp_1k_h100 + fusion MLP (lightweight) + LoRA ON (rank=128) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir real_dataset/final_training_dataset_18&19_reduceoverlap_approx1k + color_map_path configs/color_map.json + output_dir output/lora_rank_128_mlp_1k_h100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 4 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_rank_128_mlp_1k_h100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.8/188.5 GiB (6.2%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 800 files + ✓ train/seg: 800 files + ✓ train/depth: 800 files + ✓ val/rgb: 199 files + ✓ val/seg: 199 files + ✓ val/depth: 199 files + ✓ test/rgb: 85 files + ✓ test/seg: 85 files + ✓ test/depth: 85 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.7s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 12.2/188.5 GiB (6.4%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (57.4s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 13.5/188.5 GiB (7.1%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 14.6M params +Control: 0.0M params +Total trainable: 14.6M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=128, alpha=128.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 39.32M + +Parameter Statistics: + HDC²A Adapter: total=14.6M trainable=14.6M + ControlNet (frozen): total=4172.8M LoRA trainable=39.32M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 14.6M + LoRA 39.32M = 53.88M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~12500 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0347, std=0.5078 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.351268 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.84 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.035400 + depth_encoder.conv_stem.6.weight: 0.031738 + semantic_encoder.conv_stem.3.weight: 0.022583 + W_s.weight: 0.021973 + W_d.weight: 0.020386 + Test result: PASSED +[MEM @ after test] RAM: 14.2/188.5 GiB (7.5%) | VRAM: 38.0/93.1 GiB (40.9%) + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 800 samples, batch_size=4 +[Data] Train: using global text embeddings +[Data] Val: 199 samples, batch_size=4 +[Data] Test: 85 samples, batch_size=4 + +====================================================================== +Starting training: 500 epochs × 200 steps = 100000 total steps + batch_size=4, grad_accum=4, world_size=1, effective_bs=16 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 10000, 20000, 30000, 40000, 50000, 59999, 69999, 79999, 89999, 99999] + [Milestone Vis] 21 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5), test_6(5), test_7(5), test_8(5), test_9(5), test_10(5), test_11(5), test_12(5), test_13(5), test_14(5), test_15(5), test_16(5) + +--- Epoch 0/499 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_5.png + [MilestoneVis] test_6 step 0 ✓ + [MilestoneGrid] test_6 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_6.png + [MilestoneVis] test_7 step 0 ✓ + [MilestoneGrid] test_7 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_7.png + [MilestoneVis] test_8 step 0 ✓ + [MilestoneGrid] test_8 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_8.png + [MilestoneVis] test_9 step 0 ✓ + [MilestoneGrid] test_9 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_9.png + [MilestoneVis] test_10 step 0 ✓ + [MilestoneGrid] test_10 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_10.png + [MilestoneVis] test_11 step 0 ✓ + [MilestoneGrid] test_11 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_11.png + [MilestoneVis] test_12 step 0 ✓ + [MilestoneGrid] test_12 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_12.png + [MilestoneVis] test_13 step 0 ✓ + [MilestoneGrid] test_13 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_13.png + [MilestoneVis] test_14 step 0 ✓ + [MilestoneGrid] test_14 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_14.png + [MilestoneVis] test_15 step 0 ✓ + [MilestoneGrid] test_15 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_15.png + [MilestoneVis] test_16 step 0 ✓ + [MilestoneGrid] test_16 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_16.png + [Epoch 0][10/200] loss=0.669182 avg=0.770891 VRAM=38.6GiB | 0.0% done | ETA(epoch): 42857s + [Epoch 0][20/200] loss=0.746552 avg=0.779425 VRAM=38.5GiB | 0.0% done | ETA(epoch): 21495s + [Epoch 0][30/200] loss=0.673147 avg=0.766652 VRAM=38.6GiB | 0.0% done | ETA(epoch): 14285s + [Epoch 0][40/200] loss=0.686029 avg=0.761051 VRAM=38.5GiB | 0.0% done | ETA(epoch): 10613s + [Epoch 0][50/200] loss=0.837037 avg=0.763605 VRAM=38.6GiB | 0.1% done | ETA(epoch): 8358s + [Epoch 0][60/200] loss=0.686299 avg=0.773243 VRAM=38.5GiB | 0.1% done | ETA(epoch): 6810s + [Epoch 0][70/200] loss=0.680121 avg=0.771244 VRAM=38.6GiB | 0.1% done | ETA(epoch): 5667s + [Epoch 0][80/200] loss=0.889805 avg=0.768817 VRAM=38.5GiB | 0.1% done | ETA(epoch): 4776s + [Epoch 0][90/200] loss=0.768697 avg=0.775506 VRAM=38.6GiB | 0.1% done | ETA(epoch): 4053s + [Epoch 0][100/200] loss=0.855334 avg=0.774988 VRAM=38.5GiB | 0.1% done | ETA(epoch): 3449s + [Epoch 0][110/200] loss=0.917838 avg=0.774105 VRAM=38.6GiB | 0.1% done | ETA(epoch): 2930s + [Epoch 0][120/200] loss=0.885166 avg=0.771207 VRAM=38.5GiB | 0.1% done | ETA(epoch): 2476s + [Epoch 0][130/200] loss=0.744850 avg=0.767874 VRAM=38.6GiB | 0.1% done | ETA(epoch): 2071s + [Epoch 0][140/200] loss=0.705255 avg=0.766412 VRAM=38.5GiB | 0.1% done | ETA(epoch): 1705s + [Epoch 0][150/200] loss=0.769542 avg=0.762846 VRAM=38.6GiB | 0.1% done | ETA(epoch): 1371s + [Epoch 0][160/200] loss=0.760056 avg=0.762071 VRAM=38.5GiB | 0.2% done | ETA(epoch): 1061s + [Epoch 0][170/200] loss=0.697519 avg=0.758870 VRAM=38.6GiB | 0.2% done | ETA(epoch): 772s + [Epoch 0][180/200] loss=0.729932 avg=0.756326 VRAM=38.5GiB | 0.2% done | ETA(epoch): 501s + [Epoch 0][190/200] loss=0.686752 avg=0.755942 VRAM=38.6GiB | 0.2% done | ETA(epoch): 244s + [Epoch 0][200/200] loss=0.720071 avg=0.753973 VRAM=38.5GiB | 0.2% done | ETA(epoch): 0s + Train loss: 0.753973 (4775.5s) ETA: 39890min + Val loss: 0.776374 [t_0.0-0.2=0.9923 t_0.2-0.4=0.8148 t_0.4-0.6=0.6727 t_0.6-0.8=0.6093 t_0.8-1.0=0.8007] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 1/499 (0% done) --- + [Epoch 1][10/200] loss=0.590447 avg=0.721190 VRAM=38.6GiB | 0.2% done | ETA(epoch): 2518s + [Epoch 1][20/200] loss=0.692673 avg=0.727928 VRAM=38.5GiB | 0.2% done | ETA(epoch): 2388s + [Epoch 1][30/200] loss=0.727934 avg=0.734682 VRAM=38.6GiB | 0.2% done | ETA(epoch): 2254s + [Epoch 1][40/200] loss=0.716169 avg=0.716892 VRAM=38.5GiB | 0.2% done | ETA(epoch): 2121s + [Epoch 1][50/200] loss=0.585082 avg=0.709375 VRAM=38.6GiB | 0.2% done | ETA(epoch): 1991s + [Epoch 1][60/200] loss=0.658297 avg=0.716412 VRAM=38.5GiB | 0.3% done | ETA(epoch): 1858s + [Epoch 1][70/200] loss=0.568173 avg=0.714286 VRAM=38.6GiB | 0.3% done | ETA(epoch): 1725s + [Epoch 1][80/200] loss=0.647412 avg=0.715756 VRAM=38.5GiB | 0.3% done | ETA(epoch): 1592s + [Epoch 1][90/200] loss=0.747618 avg=0.715622 VRAM=38.6GiB | 0.3% done | ETA(epoch): 1459s + [Epoch 1][100/200] loss=0.643886 avg=0.714497 VRAM=38.5GiB | 0.3% done | ETA(epoch): 1326s + [Epoch 1][110/200] loss=0.812374 avg=0.712544 VRAM=38.6GiB | 0.3% done | ETA(epoch): 1194s + [Epoch 1][120/200] loss=0.737254 avg=0.709483 VRAM=38.5GiB | 0.3% done | ETA(epoch): 1061s + [Epoch 1][130/200] loss=0.621429 avg=0.711811 VRAM=38.6GiB | 0.3% done | ETA(epoch): 928s + [Epoch 1][140/200] loss=0.727495 avg=0.712663 VRAM=38.5GiB | 0.3% done | ETA(epoch): 797s + [Epoch 1][150/200] loss=0.587127 avg=0.712487 VRAM=38.6GiB | 0.4% done | ETA(epoch): 664s + [Epoch 1][160/200] loss=0.689301 avg=0.712564 VRAM=38.5GiB | 0.4% done | ETA(epoch): 531s + [Epoch 1][170/200] loss=0.667351 avg=0.711196 VRAM=38.6GiB | 0.4% done | ETA(epoch): 398s + [Epoch 1][180/200] loss=0.759977 avg=0.708885 VRAM=38.5GiB | 0.4% done | ETA(epoch): 266s + [Epoch 1][190/200] loss=0.659370 avg=0.707633 VRAM=38.6GiB | 0.4% done | ETA(epoch): 133s + [Epoch 1][200/200] loss=0.604673 avg=0.706513 VRAM=38.5GiB | 0.4% done | ETA(epoch): 0s + Train loss: 0.706513 (2655.4s) ETA: 31964min + [Epoch 2][10/200] loss=0.675212 avg=0.690239 VRAM=38.6GiB | 0.4% done | ETA(epoch): 2596s + [Epoch 2][20/200] loss=0.741553 avg=0.673390 VRAM=38.5GiB | 0.4% done | ETA(epoch): 2422s + [Epoch 2][30/200] loss=0.688834 avg=0.669291 VRAM=38.6GiB | 0.4% done | ETA(epoch): 2277s + [Epoch 2][40/200] loss=0.730349 avg=0.679017 VRAM=38.5GiB | 0.4% done | ETA(epoch): 2138s + [Epoch 2][50/200] loss=0.659107 avg=0.681458 VRAM=38.6GiB | 0.4% done | ETA(epoch): 2001s + [Epoch 2][60/200] loss=0.753742 avg=0.682912 VRAM=38.5GiB | 0.5% done | ETA(epoch): 1866s + [Epoch 2][70/200] loss=0.625165 avg=0.679021 VRAM=38.6GiB | 0.5% done | ETA(epoch): 1731s + [Epoch 2][80/200] loss=0.624456 avg=0.681715 VRAM=38.5GiB | 0.5% done | ETA(epoch): 1598s + [Epoch 2][90/200] loss=0.693469 avg=0.684182 VRAM=38.6GiB | 0.5% done | ETA(epoch): 1464s + [Epoch 2][100/200] loss=0.575366 avg=0.682433 VRAM=38.5GiB | 0.5% done | ETA(epoch): 1333s + [Epoch 2][110/200] loss=0.851435 avg=0.686562 VRAM=38.6GiB | 0.5% done | ETA(epoch): 1199s + [Epoch 2][120/200] loss=0.649544 avg=0.689361 VRAM=38.5GiB | 0.5% done | ETA(epoch): 1065s + [Epoch 2][130/200] loss=0.635451 avg=0.688461 VRAM=38.6GiB | 0.5% done | ETA(epoch): 932s + [Epoch 2][140/200] loss=0.655201 avg=0.688452 VRAM=38.5GiB | 0.5% done | ETA(epoch): 798s + [Epoch 2][150/200] loss=0.627205 avg=0.689916 VRAM=38.6GiB | 0.5% done | ETA(epoch): 665s + [Epoch 2][160/200] loss=0.776978 avg=0.690061 VRAM=38.5GiB | 0.6% done | ETA(epoch): 532s + [Epoch 2][170/200] loss=0.653651 avg=0.688282 VRAM=38.6GiB | 0.6% done | ETA(epoch): 399s + [Epoch 2][180/200] loss=0.817951 avg=0.689359 VRAM=38.5GiB | 0.6% done | ETA(epoch): 266s + [Epoch 2][190/200] loss=0.706982 avg=0.689507 VRAM=38.6GiB | 0.6% done | ETA(epoch): 133s + [Epoch 2][200/200] loss=0.624245 avg=0.689629 VRAM=38.5GiB | 0.6% done | ETA(epoch): 0s + Train loss: 0.689629 (2658.4s) ETA: 29287min + Val loss: 0.756828 [t_0.0-0.2=0.9257 t_0.2-0.4=0.7824 t_0.4-0.6=0.6683 t_0.6-0.8=0.5674 t_0.8-1.0=0.7886] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 3/499 (1% done) --- + [Epoch 3][10/200] loss=0.651568 avg=0.691741 VRAM=38.6GiB | 0.6% done | ETA(epoch): 2524s + [Epoch 3][20/200] loss=0.695919 avg=0.675775 VRAM=38.5GiB | 0.6% done | ETA(epoch): 2388s + [Epoch 3][30/200] loss=0.630110 avg=0.674217 VRAM=38.6GiB | 0.6% done | ETA(epoch): 2256s + [Epoch 3][40/200] loss=0.631442 avg=0.678834 VRAM=38.5GiB | 0.6% done | ETA(epoch): 2122s + [Epoch 3][50/200] loss=0.630832 avg=0.680661 VRAM=38.6GiB | 0.7% done | ETA(epoch): 1990s + [Epoch 3][60/200] loss=0.746439 avg=0.679009 VRAM=38.5GiB | 0.7% done | ETA(epoch): 1862s + [Epoch 3][70/200] loss=0.760932 avg=0.686273 VRAM=38.6GiB | 0.7% done | ETA(epoch): 1729s + [Epoch 3][80/200] loss=0.795391 avg=0.688284 VRAM=38.5GiB | 0.7% done | ETA(epoch): 1595s + [Epoch 3][90/200] loss=0.793050 avg=0.688826 VRAM=38.6GiB | 0.7% done | ETA(epoch): 1462s + [Epoch 3][100/200] loss=0.703547 avg=0.691691 VRAM=38.5GiB | 0.7% done | ETA(epoch): 1329s + [Epoch 3][110/200] loss=0.604521 avg=0.685969 VRAM=38.6GiB | 0.7% done | ETA(epoch): 1196s + [Epoch 3][120/200] loss=0.721282 avg=0.686243 VRAM=38.5GiB | 0.7% done | ETA(epoch): 1062s + [Epoch 3][130/200] loss=0.650721 avg=0.685047 VRAM=38.6GiB | 0.7% done | ETA(epoch): 929s + [Epoch 3][140/200] loss=0.724858 avg=0.685292 VRAM=38.5GiB | 0.7% done | ETA(epoch): 797s + [Epoch 3][150/200] loss=0.629959 avg=0.685444 VRAM=38.6GiB | 0.8% done | ETA(epoch): 664s + [Epoch 3][160/200] loss=0.634646 avg=0.686270 VRAM=38.5GiB | 0.8% done | ETA(epoch): 532s + [Epoch 3][170/200] loss=0.612847 avg=0.686968 VRAM=38.6GiB | 0.8% done | ETA(epoch): 399s + [Epoch 3][180/200] loss=0.734403 avg=0.687205 VRAM=38.5GiB | 0.8% done | ETA(epoch): 266s + [Epoch 3][190/200] loss=0.658210 avg=0.686897 VRAM=38.6GiB | 0.8% done | ETA(epoch): 133s + [Epoch 3][200/200] loss=0.642057 avg=0.685893 VRAM=38.5GiB | 0.8% done | ETA(epoch): 0s + Train loss: 0.685893 (2659.7s) ETA: 27932min + Val loss: 0.716942 [t_0.0-0.2=0.9514 t_0.2-0.4=0.7776 t_0.4-0.6=0.6529 t_0.6-0.8=0.5770 t_0.8-1.0=0.6772] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 4/499 (1% done) --- + [Epoch 4][10/200] loss=0.723340 avg=0.689342 VRAM=38.6GiB | 0.8% done | ETA(epoch): 2514s + [Epoch 4][20/200] loss=0.828632 avg=0.681501 VRAM=38.5GiB | 0.8% done | ETA(epoch): 2386s + [Epoch 4][30/200] loss=0.876533 avg=0.691933 VRAM=38.6GiB | 0.8% done | ETA(epoch): 2253s + [Epoch 4][40/200] loss=0.765088 avg=0.692643 VRAM=38.5GiB | 0.8% done | ETA(epoch): 2122s + [Epoch 4][50/200] loss=0.715889 avg=0.685487 VRAM=38.6GiB | 0.9% done | ETA(epoch): 1989s + [Epoch 4][60/200] loss=0.696134 avg=0.687632 VRAM=38.5GiB | 0.9% done | ETA(epoch): 1856s + [Epoch 4][70/200] loss=0.764015 avg=0.686716 VRAM=38.6GiB | 0.9% done | ETA(epoch): 1723s + [Epoch 4][80/200] loss=0.657223 avg=0.681992 VRAM=38.5GiB | 0.9% done | ETA(epoch): 1591s + [Epoch 4][90/200] loss=0.679451 avg=0.681537 VRAM=38.6GiB | 0.9% done | ETA(epoch): 1458s + [Epoch 4][100/200] loss=0.618022 avg=0.680592 VRAM=38.5GiB | 0.9% done | ETA(epoch): 1326s + [Epoch 4][110/200] loss=0.670005 avg=0.683690 VRAM=38.6GiB | 0.9% done | ETA(epoch): 1193s + [Epoch 4][120/200] loss=0.642359 avg=0.684965 VRAM=38.5GiB | 0.9% done | ETA(epoch): 1062s + [Epoch 4][130/200] loss=0.653820 avg=0.684763 VRAM=38.6GiB | 0.9% done | ETA(epoch): 929s + [Epoch 4][140/200] loss=0.565941 avg=0.685152 VRAM=38.5GiB | 0.9% done | ETA(epoch): 797s + [Epoch 4][150/200] loss=0.607952 avg=0.687251 VRAM=38.6GiB | 0.9% done | ETA(epoch): 664s + [Epoch 4][160/200] loss=0.641753 avg=0.685560 VRAM=38.5GiB | 1.0% done | ETA(epoch): 531s + [Epoch 4][170/200] loss=0.650371 avg=0.684498 VRAM=38.6GiB | 1.0% done | ETA(epoch): 398s + [Epoch 4][180/200] loss=0.651255 avg=0.685948 VRAM=38.5GiB | 1.0% done | ETA(epoch): 265s + [Epoch 4][190/200] loss=0.563262 avg=0.685886 VRAM=38.6GiB | 1.0% done | ETA(epoch): 133s + [Epoch 4][200/200] loss=0.656035 avg=0.685114 VRAM=38.5GiB | 1.0% done | ETA(epoch): 0s + Train loss: 0.685114 (2654.6s) ETA: 27092min + Val loss: 0.734331 [t_0.0-0.2=0.9245 t_0.2-0.4=0.7642 t_0.4-0.6=0.6728 t_0.6-0.8=0.5968 t_0.8-1.0=0.6788] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 5/499 (1% done) --- + [Epoch 5][10/200] loss=0.815073 avg=0.684254 VRAM=38.6GiB | 1.0% done | ETA(epoch): 2517s + [Epoch 5][20/200] loss=0.650669 avg=0.681538 VRAM=38.5GiB | 1.0% done | ETA(epoch): 2388s + [Epoch 5][30/200] loss=0.609538 avg=0.669979 VRAM=38.6GiB | 1.0% done | ETA(epoch): 2254s + [Epoch 5][40/200] loss=0.798949 avg=0.675921 VRAM=38.5GiB | 1.0% done | ETA(epoch): 2122s + [Epoch 5][50/200] loss=0.604868 avg=0.680017 VRAM=38.6GiB | 1.1% done | ETA(epoch): 1988s + [Epoch 5][60/200] loss=0.606695 avg=0.674532 VRAM=38.5GiB | 1.1% done | ETA(epoch): 1856s + [Epoch 5][70/200] loss=0.641906 avg=0.671436 VRAM=38.6GiB | 1.1% done | ETA(epoch): 1724s + [Epoch 5][80/200] loss=0.679614 avg=0.675605 VRAM=38.5GiB | 1.1% done | ETA(epoch): 1603s + [Epoch 5][90/200] loss=0.691954 avg=0.681072 VRAM=38.6GiB | 1.1% done | ETA(epoch): 1468s + [Epoch 5][100/200] loss=0.765461 avg=0.679373 VRAM=38.5GiB | 1.1% done | ETA(epoch): 1334s + [Epoch 5][110/200] loss=0.618238 avg=0.680767 VRAM=38.6GiB | 1.1% done | ETA(epoch): 1199s + [Epoch 5][120/200] loss=0.649960 avg=0.681715 VRAM=38.5GiB | 1.1% done | ETA(epoch): 1066s + [Epoch 5][130/200] loss=0.645614 avg=0.684072 VRAM=38.6GiB | 1.1% done | ETA(epoch): 932s + [Epoch 5][140/200] loss=0.641811 avg=0.684595 VRAM=38.5GiB | 1.1% done | ETA(epoch): 799s + [Epoch 5][150/200] loss=0.704240 avg=0.685560 VRAM=38.6GiB | 1.1% done | ETA(epoch): 665s + [Epoch 5][160/200] loss=0.687591 avg=0.686871 VRAM=38.5GiB | 1.2% done | ETA(epoch): 532s + [Epoch 5][170/200] loss=0.782543 avg=0.688260 VRAM=38.6GiB | 1.2% done | ETA(epoch): 399s + [Epoch 5][180/200] loss=0.564997 avg=0.685382 VRAM=38.5GiB | 1.2% done | ETA(epoch): 266s + [Epoch 5][190/200] loss=0.650636 avg=0.684639 VRAM=38.6GiB | 1.2% done | ETA(epoch): 133s + [Epoch 5][200/200] loss=0.629247 avg=0.683529 VRAM=38.5GiB | 1.2% done | ETA(epoch): 0s + Train loss: 0.683529 (2659.3s) ETA: 26526min + Val loss: 0.705552 [t_0.0-0.2=0.9770 t_0.2-0.4=0.7678 t_0.4-0.6=0.6588 t_0.6-0.8=0.5857 t_0.8-1.0=0.6366] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0005 (BEST) + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 22.0/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 6/499 (1% done) --- + [Epoch 6][10/200] loss=0.666060 avg=0.680888 VRAM=38.6GiB | 1.2% done | ETA(epoch): 2524s + [Epoch 6][20/200] loss=0.666562 avg=0.661158 VRAM=38.5GiB | 1.2% done | ETA(epoch): 2388s + [Epoch 6][30/200] loss=0.667458 avg=0.672141 VRAM=38.6GiB | 1.2% done | ETA(epoch): 2256s + [Epoch 6][40/200] loss=0.803178 avg=0.678526 VRAM=38.5GiB | 1.2% done | ETA(epoch): 2132s + [Epoch 6][50/200] loss=0.670168 avg=0.673615 VRAM=38.6GiB | 1.2% done | ETA(epoch): 1996s + [Epoch 6][60/200] loss=0.661248 avg=0.675629 VRAM=38.5GiB | 1.3% done | ETA(epoch): 1862s + [Epoch 6][70/200] loss=0.701192 avg=0.669372 VRAM=38.6GiB | 1.3% done | ETA(epoch): 1728s + [Epoch 6][80/200] loss=0.603179 avg=0.668654 VRAM=38.5GiB | 1.3% done | ETA(epoch): 1594s + [Epoch 6][90/200] loss=0.764719 avg=0.673710 VRAM=38.6GiB | 1.3% done | ETA(epoch): 1461s + [Epoch 6][100/200] loss=0.813260 avg=0.674702 VRAM=38.5GiB | 1.3% done | ETA(epoch): 1328s + [Epoch 6][110/200] loss=0.694442 avg=0.676160 VRAM=38.6GiB | 1.3% done | ETA(epoch): 1195s + [Epoch 6][120/200] loss=0.636292 avg=0.676789 VRAM=38.5GiB | 1.3% done | ETA(epoch): 1062s + [Epoch 6][130/200] loss=0.696547 avg=0.677029 VRAM=38.6GiB | 1.3% done | ETA(epoch): 931s + [Epoch 6][140/200] loss=0.681483 avg=0.675935 VRAM=38.5GiB | 1.3% done | ETA(epoch): 798s + [Epoch 6][150/200] loss=0.691938 avg=0.677633 VRAM=38.6GiB | 1.4% done | ETA(epoch): 665s + [Epoch 6][160/200] loss=0.574300 avg=0.679106 VRAM=38.5GiB | 1.4% done | ETA(epoch): 532s + [Epoch 6][170/200] loss=0.738485 avg=0.677609 VRAM=38.6GiB | 1.4% done | ETA(epoch): 399s + [Epoch 6][180/200] loss=0.681078 avg=0.676371 VRAM=38.5GiB | 1.4% done | ETA(epoch): 266s + [Epoch 6][190/200] loss=0.710825 avg=0.677669 VRAM=38.6GiB | 1.4% done | ETA(epoch): 133s + [Epoch 6][200/200] loss=0.596381 avg=0.677132 VRAM=38.5GiB | 1.4% done | ETA(epoch): 0s + Train loss: 0.677132 (2656.8s) ETA: 26101min + Val loss: 0.708566 [t_0.0-0.2=0.9599 t_0.2-0.4=0.7758 t_0.4-0.6=0.6426 t_0.6-0.8=0.5702 t_0.8-1.0=0.6420] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0006 + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 7/499 (1% done) --- + [Epoch 7][10/200] loss=0.625892 avg=0.681395 VRAM=38.6GiB | 1.4% done | ETA(epoch): 2515s + [Epoch 7][20/200] loss=0.611689 avg=0.699253 VRAM=38.5GiB | 1.4% done | ETA(epoch): 2385s + [Epoch 7][30/200] loss=0.616209 avg=0.695299 VRAM=38.6GiB | 1.4% done | ETA(epoch): 2253s + [Epoch 7][40/200] loss=0.644874 avg=0.688678 VRAM=38.5GiB | 1.4% done | ETA(epoch): 2120s + [Epoch 7][50/200] loss=0.856091 avg=0.688608 VRAM=38.6GiB | 1.5% done | ETA(epoch): 1988s + [Epoch 7][60/200] loss=0.703903 avg=0.677680 VRAM=38.5GiB | 1.5% done | ETA(epoch): 1856s + [Epoch 7][70/200] loss=0.675584 avg=0.679304 VRAM=38.6GiB | 1.5% done | ETA(epoch): 1723s + [Epoch 7][80/200] loss=0.696602 avg=0.682154 VRAM=38.5GiB | 1.5% done | ETA(epoch): 1591s + [Epoch 7][90/200] loss=0.855880 avg=0.688932 VRAM=38.6GiB | 1.5% done | ETA(epoch): 1459s + [Epoch 7][100/200] loss=0.605212 avg=0.689972 VRAM=38.5GiB | 1.5% done | ETA(epoch): 1329s + [Epoch 7][110/200] loss=0.643707 avg=0.687855 VRAM=38.6GiB | 1.5% done | ETA(epoch): 1195s + [Epoch 7][120/200] loss=0.662097 avg=0.689211 VRAM=38.5GiB | 1.5% done | ETA(epoch): 1063s + [Epoch 7][130/200] loss=0.623736 avg=0.687494 VRAM=38.6GiB | 1.5% done | ETA(epoch): 930s + [Epoch 7][140/200] loss=0.607969 avg=0.687543 VRAM=38.5GiB | 1.5% done | ETA(epoch): 797s + [Epoch 7][150/200] loss=0.716622 avg=0.688388 VRAM=38.6GiB | 1.6% done | ETA(epoch): 664s + [Epoch 7][160/200] loss=0.729278 avg=0.688528 VRAM=38.5GiB | 1.6% done | ETA(epoch): 531s + [Epoch 7][170/200] loss=0.723192 avg=0.688144 VRAM=38.6GiB | 1.6% done | ETA(epoch): 398s + [Epoch 7][180/200] loss=0.610126 avg=0.688012 VRAM=38.5GiB | 1.6% done | ETA(epoch): 265s + [Epoch 7][190/200] loss=0.624421 avg=0.688609 VRAM=38.6GiB | 1.6% done | ETA(epoch): 133s + [Epoch 7][200/200] loss=0.628225 avg=0.688551 VRAM=38.5GiB | 1.6% done | ETA(epoch): 0s + Train loss: 0.688551 (2654.5s) ETA: 25768min + Val loss: 0.716531 [t_0.0-0.2=0.9326 t_0.2-0.4=0.7649 t_0.4-0.6=0.6468 t_0.6-0.8=0.6000 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 8/499 (2% done) --- + [Epoch 8][10/200] loss=0.666818 avg=0.699338 VRAM=38.6GiB | 1.6% done | ETA(epoch): 2521s + [Epoch 8][20/200] loss=0.680953 avg=0.693299 VRAM=38.5GiB | 1.6% done | ETA(epoch): 2387s + [Epoch 8][30/200] loss=0.705965 avg=0.687063 VRAM=38.6GiB | 1.6% done | ETA(epoch): 2254s + [Epoch 8][40/200] loss=0.677591 avg=0.695430 VRAM=38.5GiB | 1.6% done | ETA(epoch): 2121s + [Epoch 8][50/200] loss=0.766927 avg=0.699206 VRAM=38.6GiB | 1.7% done | ETA(epoch): 1989s + [Epoch 8][60/200] loss=0.651186 avg=0.693229 VRAM=38.5GiB | 1.7% done | ETA(epoch): 1863s + [Epoch 8][70/200] loss=0.656270 avg=0.695041 VRAM=38.6GiB | 1.7% done | ETA(epoch): 1729s + [Epoch 8][80/200] loss=0.638704 avg=0.691069 VRAM=38.5GiB | 1.7% done | ETA(epoch): 1596s + [Epoch 8][90/200] loss=0.642327 avg=0.693040 VRAM=38.6GiB | 1.7% done | ETA(epoch): 1462s + [Epoch 8][100/200] loss=0.626093 avg=0.691618 VRAM=38.5GiB | 1.7% done | ETA(epoch): 1329s + [Epoch 8][110/200] loss=0.722749 avg=0.692318 VRAM=38.6GiB | 1.7% done | ETA(epoch): 1196s + [Epoch 8][120/200] loss=0.564535 avg=0.688448 VRAM=38.5GiB | 1.7% done | ETA(epoch): 1063s + [Epoch 8][130/200] loss=0.680423 avg=0.687100 VRAM=38.6GiB | 1.7% done | ETA(epoch): 930s + [Epoch 8][140/200] loss=0.643521 avg=0.683411 VRAM=38.5GiB | 1.7% done | ETA(epoch): 797s + [Epoch 8][150/200] loss=0.578142 avg=0.682254 VRAM=38.6GiB | 1.8% done | ETA(epoch): 664s + [Epoch 8][160/200] loss=0.797981 avg=0.681871 VRAM=38.5GiB | 1.8% done | ETA(epoch): 531s + [Epoch 8][170/200] loss=0.755247 avg=0.683668 VRAM=38.6GiB | 1.8% done | ETA(epoch): 399s + [Epoch 8][180/200] loss=0.666021 avg=0.683000 VRAM=38.5GiB | 1.8% done | ETA(epoch): 266s + [Epoch 8][190/200] loss=0.599412 avg=0.682322 VRAM=38.6GiB | 1.8% done | ETA(epoch): 133s + [Epoch 8][200/200] loss=0.624539 avg=0.680907 VRAM=38.5GiB | 1.8% done | ETA(epoch): 0s + Train loss: 0.680907 (2656.1s) ETA: 25500min + Val loss: 0.718692 [t_0.0-0.2=0.9348 t_0.2-0.4=0.7683 t_0.4-0.6=0.6620 t_0.6-0.8=0.5753 t_0.8-1.0=0.6367] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0008 +[MEM @ epoch 8 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 9/499 (2% done) --- + [Epoch 9][10/200] loss=0.758457 avg=0.708241 VRAM=38.6GiB | 1.8% done | ETA(epoch): 2518s + [Epoch 9][20/200] loss=0.575039 avg=0.686417 VRAM=38.5GiB | 1.8% done | ETA(epoch): 2389s + [Epoch 9][30/200] loss=0.616877 avg=0.699021 VRAM=38.6GiB | 1.8% done | ETA(epoch): 2256s + [Epoch 9][40/200] loss=0.681667 avg=0.689184 VRAM=38.5GiB | 1.8% done | ETA(epoch): 2123s + [Epoch 9][50/200] loss=0.590068 avg=0.686042 VRAM=38.6GiB | 1.8% done | ETA(epoch): 1990s + [Epoch 9][60/200] loss=0.624416 avg=0.681194 VRAM=38.5GiB | 1.9% done | ETA(epoch): 1857s + [Epoch 9][70/200] loss=0.650123 avg=0.682230 VRAM=38.6GiB | 1.9% done | ETA(epoch): 1725s + [Epoch 9][80/200] loss=0.722138 avg=0.687303 VRAM=38.5GiB | 1.9% done | ETA(epoch): 1592s + [Epoch 9][90/200] loss=0.779318 avg=0.684309 VRAM=38.6GiB | 1.9% done | ETA(epoch): 1459s + [Epoch 9][100/200] loss=0.598044 avg=0.680469 VRAM=38.5GiB | 1.9% done | ETA(epoch): 1327s + [Epoch 9][110/200] loss=0.660282 avg=0.678080 VRAM=38.6GiB | 1.9% done | ETA(epoch): 1194s + [Epoch 9][120/200] loss=0.722315 avg=0.676180 VRAM=38.5GiB | 1.9% done | ETA(epoch): 1061s + [Epoch 9][130/200] loss=0.641014 avg=0.676541 VRAM=38.6GiB | 1.9% done | ETA(epoch): 928s + [Epoch 9][140/200] loss=0.593320 avg=0.675338 VRAM=38.5GiB | 1.9% done | ETA(epoch): 796s + [Epoch 9][150/200] loss=0.695195 avg=0.677607 VRAM=38.6GiB | 1.9% done | ETA(epoch): 663s + [Epoch 9][160/200] loss=0.679938 avg=0.676821 VRAM=38.5GiB | 2.0% done | ETA(epoch): 530s + [Epoch 9][170/200] loss=0.615613 avg=0.676869 VRAM=38.6GiB | 2.0% done | ETA(epoch): 398s + [Epoch 9][180/200] loss=0.686119 avg=0.675708 VRAM=38.5GiB | 2.0% done | ETA(epoch): 265s + [Epoch 9][190/200] loss=0.650927 avg=0.676449 VRAM=38.6GiB | 2.0% done | ETA(epoch): 133s + [Epoch 9][200/200] loss=0.583322 avg=0.676303 VRAM=38.5GiB | 2.0% done | ETA(epoch): 0s + Train loss: 0.676303 (2652.0s) ETA: 25272min + Val loss: 0.714681 [t_0.0-0.2=0.9537 t_0.2-0.4=0.7785 t_0.4-0.6=0.6423 t_0.6-0.8=0.5432 t_0.8-1.0=0.6378] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 10/499 (2% done) --- + [Epoch 10][10/200] loss=0.587072 avg=0.661850 VRAM=38.6GiB | 2.0% done | ETA(epoch): 2524s + [Epoch 10][20/200] loss=0.800293 avg=0.656817 VRAM=38.5GiB | 2.0% done | ETA(epoch): 2387s + [Epoch 10][30/200] loss=0.582716 avg=0.674122 VRAM=38.6GiB | 2.0% done | ETA(epoch): 2254s + [Epoch 10][40/200] loss=0.670886 avg=0.670713 VRAM=38.5GiB | 2.0% done | ETA(epoch): 2122s + [Epoch 10][50/200] loss=0.777108 avg=0.669385 VRAM=38.6GiB | 2.1% done | ETA(epoch): 1989s + [Epoch 10][60/200] loss=0.704071 avg=0.668561 VRAM=38.5GiB | 2.1% done | ETA(epoch): 1857s + [Epoch 10][70/200] loss=0.737641 avg=0.666355 VRAM=38.6GiB | 2.1% done | ETA(epoch): 1724s + [Epoch 10][80/200] loss=0.595327 avg=0.664603 VRAM=38.5GiB | 2.1% done | ETA(epoch): 1592s + [Epoch 10][90/200] loss=0.707265 avg=0.665903 VRAM=38.6GiB | 2.1% done | ETA(epoch): 1459s + [Epoch 10][100/200] loss=0.696387 avg=0.665894 VRAM=38.5GiB | 2.1% done | ETA(epoch): 1326s + [Epoch 10][110/200] loss=0.610606 avg=0.664872 VRAM=38.6GiB | 2.1% done | ETA(epoch): 1194s + [Epoch 10][120/200] loss=0.668890 avg=0.667071 VRAM=38.5GiB | 2.1% done | ETA(epoch): 1061s + [Epoch 10][130/200] loss=0.653877 avg=0.667233 VRAM=38.6GiB | 2.1% done | ETA(epoch): 928s + [Epoch 10][140/200] loss=0.660274 avg=0.669548 VRAM=38.5GiB | 2.1% done | ETA(epoch): 796s + [Epoch 10][150/200] loss=0.818027 avg=0.669101 VRAM=38.6GiB | 2.1% done | ETA(epoch): 663s + [Epoch 10][160/200] loss=0.648013 avg=0.668848 VRAM=38.5GiB | 2.2% done | ETA(epoch): 531s + [Epoch 10][170/200] loss=0.661662 avg=0.668832 VRAM=38.6GiB | 2.2% done | ETA(epoch): 399s + [Epoch 10][180/200] loss=0.632915 avg=0.668589 VRAM=38.5GiB | 2.2% done | ETA(epoch): 266s + [Epoch 10][190/200] loss=0.564152 avg=0.666904 VRAM=38.6GiB | 2.2% done | ETA(epoch): 133s + [Epoch 10][200/200] loss=0.643763 avg=0.666905 VRAM=38.5GiB | 2.2% done | ETA(epoch): 0s + Train loss: 0.666905 (2656.3s) ETA: 25081min + Val loss: 0.714259 [t_0.0-0.2=0.9409 t_0.2-0.4=0.7624 t_0.4-0.6=0.6435 t_0.6-0.8=0.5785 t_0.8-1.0=0.6303] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0010 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 11/499 (2% done) --- + [Epoch 11][10/200] loss=0.718840 avg=0.681015 VRAM=38.6GiB | 2.2% done | ETA(epoch): 2515s + [Epoch 11][20/200] loss=0.701615 avg=0.689327 VRAM=38.5GiB | 2.2% done | ETA(epoch): 2387s + [Epoch 11][30/200] loss=0.627189 avg=0.678908 VRAM=38.6GiB | 2.2% done | ETA(epoch): 2253s + [Epoch 11][40/200] loss=0.678417 avg=0.680558 VRAM=38.5GiB | 2.2% done | ETA(epoch): 2134s + [Epoch 11][50/200] loss=0.659019 avg=0.685668 VRAM=38.6GiB | 2.2% done | ETA(epoch): 1998s + [Epoch 11][60/200] loss=0.626407 avg=0.686597 VRAM=38.5GiB | 2.3% done | ETA(epoch): 1864s + [Epoch 11][70/200] loss=0.592248 avg=0.681387 VRAM=38.6GiB | 2.3% done | ETA(epoch): 1730s + [Epoch 11][80/200] loss=0.623803 avg=0.684019 VRAM=38.5GiB | 2.3% done | ETA(epoch): 1596s + [Epoch 11][90/200] loss=0.704455 avg=0.684887 VRAM=38.6GiB | 2.3% done | ETA(epoch): 1463s + [Epoch 11][100/200] loss=0.627311 avg=0.683951 VRAM=38.5GiB | 2.3% done | ETA(epoch): 1329s + [Epoch 11][110/200] loss=0.670182 avg=0.680970 VRAM=38.6GiB | 2.3% done | ETA(epoch): 1196s + [Epoch 11][120/200] loss=0.668458 avg=0.682461 VRAM=38.5GiB | 2.3% done | ETA(epoch): 1063s + [Epoch 11][130/200] loss=0.669632 avg=0.678872 VRAM=38.6GiB | 2.3% done | ETA(epoch): 931s + [Epoch 11][140/200] loss=0.581282 avg=0.675132 VRAM=38.5GiB | 2.3% done | ETA(epoch): 798s + [Epoch 11][150/200] loss=0.716164 avg=0.673332 VRAM=38.6GiB | 2.4% done | ETA(epoch): 665s + [Epoch 11][160/200] loss=0.664277 avg=0.673941 VRAM=38.5GiB | 2.4% done | ETA(epoch): 532s + [Epoch 11][170/200] loss=0.736960 avg=0.675070 VRAM=38.6GiB | 2.4% done | ETA(epoch): 399s + [Epoch 11][180/200] loss=0.644012 avg=0.675517 VRAM=38.5GiB | 2.4% done | ETA(epoch): 266s + [Epoch 11][190/200] loss=0.653444 avg=0.674902 VRAM=38.6GiB | 2.4% done | ETA(epoch): 133s + [Epoch 11][200/200] loss=0.776034 avg=0.673405 VRAM=38.5GiB | 2.4% done | ETA(epoch): 0s + Train loss: 0.673405 (2657.4s) ETA: 24914min + Val loss: 0.732139 [t_0.0-0.2=0.9491 t_0.2-0.4=0.7766 t_0.4-0.6=0.6454 t_0.6-0.8=0.5679 t_0.8-1.0=0.6146] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 12/499 (2% done) --- + [Epoch 12][10/200] loss=0.650271 avg=0.655615 VRAM=38.6GiB | 2.4% done | ETA(epoch): 2517s + [Epoch 12][20/200] loss=0.900266 avg=0.681750 VRAM=38.5GiB | 2.4% done | ETA(epoch): 2385s + [Epoch 12][30/200] loss=0.701106 avg=0.685448 VRAM=38.6GiB | 2.4% done | ETA(epoch): 2255s + [Epoch 12][40/200] loss=0.649138 avg=0.673263 VRAM=38.5GiB | 2.4% done | ETA(epoch): 2123s + [Epoch 12][50/200] loss=0.709011 avg=0.670788 VRAM=38.6GiB | 2.5% done | ETA(epoch): 1990s + [Epoch 12][60/200] loss=0.608775 avg=0.668515 VRAM=38.5GiB | 2.5% done | ETA(epoch): 1858s + [Epoch 12][70/200] loss=0.590554 avg=0.668577 VRAM=38.6GiB | 2.5% done | ETA(epoch): 1724s + [Epoch 12][80/200] loss=0.611528 avg=0.665876 VRAM=38.5GiB | 2.5% done | ETA(epoch): 1592s + [Epoch 12][90/200] loss=0.613715 avg=0.668364 VRAM=38.6GiB | 2.5% done | ETA(epoch): 1459s + [Epoch 12][100/200] loss=0.657140 avg=0.664210 VRAM=38.5GiB | 2.5% done | ETA(epoch): 1326s + [Epoch 12][110/200] loss=0.712934 avg=0.665324 VRAM=38.6GiB | 2.5% done | ETA(epoch): 1194s + [Epoch 12][120/200] loss=0.541455 avg=0.663710 VRAM=38.5GiB | 2.5% done | ETA(epoch): 1061s + [Epoch 12][130/200] loss=0.696271 avg=0.662979 VRAM=38.6GiB | 2.5% done | ETA(epoch): 928s + [Epoch 12][140/200] loss=0.601559 avg=0.664387 VRAM=38.5GiB | 2.5% done | ETA(epoch): 796s + [Epoch 12][150/200] loss=0.651247 avg=0.662711 VRAM=38.6GiB | 2.5% done | ETA(epoch): 663s + [Epoch 12][160/200] loss=0.618704 avg=0.662969 VRAM=38.5GiB | 2.6% done | ETA(epoch): 531s + [Epoch 12][170/200] loss=0.660740 avg=0.663304 VRAM=38.6GiB | 2.6% done | ETA(epoch): 398s + [Epoch 12][180/200] loss=0.696147 avg=0.663513 VRAM=38.5GiB | 2.6% done | ETA(epoch): 265s + [Epoch 12][190/200] loss=0.764988 avg=0.665004 VRAM=38.6GiB | 2.6% done | ETA(epoch): 133s + [Epoch 12][200/200] loss=0.731566 avg=0.664288 VRAM=38.5GiB | 2.6% done | ETA(epoch): 0s + Train loss: 0.664288 (2653.1s) ETA: 24763min + Val loss: 0.696918 [t_0.0-0.2=0.9498 t_0.2-0.4=0.7547 t_0.4-0.6=0.6697 t_0.6-0.8=0.5780 t_0.8-1.0=0.6047] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0012 (BEST) + Deleted old checkpoint: checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 13/499 (3% done) --- + [Epoch 13][10/200] loss=0.628621 avg=0.674878 VRAM=38.6GiB | 2.6% done | ETA(epoch): 2521s + [Epoch 13][20/200] loss=0.780831 avg=0.677693 VRAM=38.5GiB | 2.6% done | ETA(epoch): 2389s + [Epoch 13][30/200] loss=0.556462 avg=0.676784 VRAM=38.6GiB | 2.6% done | ETA(epoch): 2259s + [Epoch 13][40/200] loss=0.592864 avg=0.671274 VRAM=38.5GiB | 2.6% done | ETA(epoch): 2125s + [Epoch 13][50/200] loss=0.756581 avg=0.674209 VRAM=38.6GiB | 2.6% done | ETA(epoch): 1992s + [Epoch 13][60/200] loss=0.625107 avg=0.672030 VRAM=38.5GiB | 2.7% done | ETA(epoch): 1869s + [Epoch 13][70/200] loss=0.619632 avg=0.670955 VRAM=38.6GiB | 2.7% done | ETA(epoch): 1734s + [Epoch 13][80/200] loss=0.560619 avg=0.670403 VRAM=38.5GiB | 2.7% done | ETA(epoch): 1600s + [Epoch 13][90/200] loss=0.692564 avg=0.671377 VRAM=38.6GiB | 2.7% done | ETA(epoch): 1466s + [Epoch 13][100/200] loss=0.587595 avg=0.673695 VRAM=38.5GiB | 2.7% done | ETA(epoch): 1332s + [Epoch 13][110/200] loss=0.633084 avg=0.673946 VRAM=38.6GiB | 2.7% done | ETA(epoch): 1198s + [Epoch 13][120/200] loss=0.621975 avg=0.673108 VRAM=38.5GiB | 2.7% done | ETA(epoch): 1065s + [Epoch 13][130/200] loss=0.641439 avg=0.671193 VRAM=38.6GiB | 2.7% done | ETA(epoch): 932s + [Epoch 13][140/200] loss=0.673670 avg=0.671904 VRAM=38.5GiB | 2.7% done | ETA(epoch): 798s + [Epoch 13][150/200] loss=0.631196 avg=0.672453 VRAM=38.6GiB | 2.8% done | ETA(epoch): 667s + [Epoch 13][160/200] loss=0.696553 avg=0.670707 VRAM=38.5GiB | 2.8% done | ETA(epoch): 533s + [Epoch 13][170/200] loss=0.737053 avg=0.671526 VRAM=38.6GiB | 2.8% done | ETA(epoch): 400s + [Epoch 13][180/200] loss=0.609251 avg=0.669470 VRAM=38.5GiB | 2.8% done | ETA(epoch): 266s + [Epoch 13][190/200] loss=0.684837 avg=0.670382 VRAM=38.6GiB | 2.8% done | ETA(epoch): 133s + [Epoch 13][200/200] loss=0.624851 avg=0.671228 VRAM=38.5GiB | 2.8% done | ETA(epoch): 0s + Train loss: 0.671228 (2663.6s) ETA: 24632min + Val loss: 0.717107 [t_0.0-0.2=0.9494 t_0.2-0.4=0.7524 t_0.4-0.6=0.6510 t_0.6-0.8=0.5818 t_0.8-1.0=0.6254] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 14/499 (3% done) --- + [Epoch 14][10/200] loss=0.685526 avg=0.696600 VRAM=38.6GiB | 2.8% done | ETA(epoch): 2525s + [Epoch 14][20/200] loss=0.569076 avg=0.671492 VRAM=38.5GiB | 2.8% done | ETA(epoch): 2388s + [Epoch 14][30/200] loss=0.690147 avg=0.666452 VRAM=38.6GiB | 2.8% done | ETA(epoch): 2255s + [Epoch 14][40/200] loss=0.662858 avg=0.662408 VRAM=38.5GiB | 2.8% done | ETA(epoch): 2122s + [Epoch 14][50/200] loss=0.629352 avg=0.660539 VRAM=38.6GiB | 2.9% done | ETA(epoch): 1989s + [Epoch 14][60/200] loss=0.664640 avg=0.658462 VRAM=38.5GiB | 2.9% done | ETA(epoch): 1856s + [Epoch 14][70/200] loss=0.828582 avg=0.664354 VRAM=38.6GiB | 2.9% done | ETA(epoch): 1724s + [Epoch 14][80/200] loss=0.634735 avg=0.666654 VRAM=38.5GiB | 2.9% done | ETA(epoch): 1591s + [Epoch 14][90/200] loss=0.619165 avg=0.663136 VRAM=38.6GiB | 2.9% done | ETA(epoch): 1458s + [Epoch 14][100/200] loss=0.753792 avg=0.664222 VRAM=38.5GiB | 2.9% done | ETA(epoch): 1326s + [Epoch 14][110/200] loss=0.678785 avg=0.667113 VRAM=38.6GiB | 2.9% done | ETA(epoch): 1193s + [Epoch 14][120/200] loss=0.645343 avg=0.667106 VRAM=38.5GiB | 2.9% done | ETA(epoch): 1061s + [Epoch 14][130/200] loss=0.784880 avg=0.667486 VRAM=38.6GiB | 2.9% done | ETA(epoch): 928s + [Epoch 14][140/200] loss=0.770630 avg=0.666775 VRAM=38.5GiB | 2.9% done | ETA(epoch): 795s + [Epoch 14][150/200] loss=0.717971 avg=0.669022 VRAM=38.6GiB | 2.9% done | ETA(epoch): 663s + [Epoch 14][160/200] loss=0.578235 avg=0.670370 VRAM=38.5GiB | 3.0% done | ETA(epoch): 530s + [Epoch 14][170/200] loss=0.775529 avg=0.670183 VRAM=38.6GiB | 3.0% done | ETA(epoch): 398s + [Epoch 14][180/200] loss=0.633418 avg=0.669200 VRAM=38.5GiB | 3.0% done | ETA(epoch): 265s + [Epoch 14][190/200] loss=0.611906 avg=0.668809 VRAM=38.6GiB | 3.0% done | ETA(epoch): 133s + [Epoch 14][200/200] loss=0.618068 avg=0.667836 VRAM=38.5GiB | 3.0% done | ETA(epoch): 0s + Train loss: 0.667836 (2654.4s) ETA: 24507min + Val loss: 0.685863 [t_0.0-0.2=0.9332 t_0.2-0.4=0.7574 t_0.4-0.6=0.6611 t_0.6-0.8=0.5584 t_0.8-1.0=0.6209] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0014 (BEST) + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 20.8/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 15/499 (3% done) --- + [Epoch 15][10/200] loss=0.637046 avg=0.655527 VRAM=38.6GiB | 3.0% done | ETA(epoch): 2519s + [Epoch 15][20/200] loss=0.776584 avg=0.652852 VRAM=38.5GiB | 3.0% done | ETA(epoch): 2387s + [Epoch 15][30/200] loss=0.745512 avg=0.656270 VRAM=38.6GiB | 3.0% done | ETA(epoch): 2254s + [Epoch 15][40/200] loss=0.648695 avg=0.658711 VRAM=38.5GiB | 3.0% done | ETA(epoch): 2123s + [Epoch 15][50/200] loss=0.558707 avg=0.656215 VRAM=38.6GiB | 3.0% done | ETA(epoch): 1991s + [Epoch 15][60/200] loss=0.548102 avg=0.654230 VRAM=38.5GiB | 3.1% done | ETA(epoch): 1857s + [Epoch 15][70/200] loss=0.618626 avg=0.656083 VRAM=38.6GiB | 3.1% done | ETA(epoch): 1725s + [Epoch 15][80/200] loss=0.573328 avg=0.656907 VRAM=38.5GiB | 3.1% done | ETA(epoch): 1592s + [Epoch 15][90/200] loss=0.641863 avg=0.656223 VRAM=38.6GiB | 3.1% done | ETA(epoch): 1459s + [Epoch 15][100/200] loss=0.750535 avg=0.656350 VRAM=38.5GiB | 3.1% done | ETA(epoch): 1326s + [Epoch 15][110/200] loss=0.670692 avg=0.659934 VRAM=38.6GiB | 3.1% done | ETA(epoch): 1194s + [Epoch 15][120/200] loss=0.707462 avg=0.658726 VRAM=38.5GiB | 3.1% done | ETA(epoch): 1061s + [Epoch 15][130/200] loss=0.695660 avg=0.663385 VRAM=38.6GiB | 3.1% done | ETA(epoch): 928s + [Epoch 15][140/200] loss=0.714177 avg=0.662431 VRAM=38.5GiB | 3.1% done | ETA(epoch): 796s + [Epoch 15][150/200] loss=0.653016 avg=0.662551 VRAM=38.6GiB | 3.1% done | ETA(epoch): 663s + [Epoch 15][160/200] loss=0.647310 avg=0.662191 VRAM=38.5GiB | 3.2% done | ETA(epoch): 531s + [Epoch 15][170/200] loss=0.556168 avg=0.662314 VRAM=38.6GiB | 3.2% done | ETA(epoch): 399s + [Epoch 15][180/200] loss=0.557230 avg=0.661317 VRAM=38.5GiB | 3.2% done | ETA(epoch): 266s + [Epoch 15][190/200] loss=0.783740 avg=0.663392 VRAM=38.6GiB | 3.2% done | ETA(epoch): 133s + [Epoch 15][200/200] loss=0.681449 avg=0.663934 VRAM=38.5GiB | 3.2% done | ETA(epoch): 0s + Train loss: 0.663934 (2656.1s) ETA: 24393min + Val loss: 0.699542 [t_0.0-0.2=0.9368 t_0.2-0.4=0.7613 t_0.4-0.6=0.6499 t_0.6-0.8=0.5755 t_0.8-1.0=0.6176] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 16/499 (3% done) --- + [Epoch 16][10/200] loss=0.612524 avg=0.671042 VRAM=38.6GiB | 3.2% done | ETA(epoch): 2516s + [Epoch 16][20/200] loss=0.686797 avg=0.680599 VRAM=38.5GiB | 3.2% done | ETA(epoch): 2386s + [Epoch 16][30/200] loss=0.595542 avg=0.683480 VRAM=38.6GiB | 3.2% done | ETA(epoch): 2255s + [Epoch 16][40/200] loss=0.719969 avg=0.676010 VRAM=38.5GiB | 3.2% done | ETA(epoch): 2122s + [Epoch 16][50/200] loss=0.666175 avg=0.667190 VRAM=38.6GiB | 3.2% done | ETA(epoch): 1990s + [Epoch 16][60/200] loss=0.732959 avg=0.664460 VRAM=38.5GiB | 3.3% done | ETA(epoch): 1857s + [Epoch 16][70/200] loss=0.585803 avg=0.666733 VRAM=38.6GiB | 3.3% done | ETA(epoch): 1724s + [Epoch 16][80/200] loss=0.611725 avg=0.665589 VRAM=38.5GiB | 3.3% done | ETA(epoch): 1591s + [Epoch 16][90/200] loss=0.694382 avg=0.662846 VRAM=38.6GiB | 3.3% done | ETA(epoch): 1459s + [Epoch 16][100/200] loss=0.578678 avg=0.663243 VRAM=38.5GiB | 3.3% done | ETA(epoch): 1326s + [Epoch 16][110/200] loss=0.653834 avg=0.663081 VRAM=38.6GiB | 3.3% done | ETA(epoch): 1194s + [Epoch 16][120/200] loss=0.687554 avg=0.664931 VRAM=38.5GiB | 3.3% done | ETA(epoch): 1061s + [Epoch 16][130/200] loss=0.828014 avg=0.667549 VRAM=38.6GiB | 3.3% done | ETA(epoch): 928s + [Epoch 16][140/200] loss=0.558386 avg=0.664035 VRAM=38.5GiB | 3.3% done | ETA(epoch): 796s + [Epoch 16][150/200] loss=0.600046 avg=0.664020 VRAM=38.6GiB | 3.4% done | ETA(epoch): 663s + [Epoch 16][160/200] loss=0.698340 avg=0.663456 VRAM=38.5GiB | 3.4% done | ETA(epoch): 530s + [Epoch 16][170/200] loss=0.648382 avg=0.665200 VRAM=38.6GiB | 3.4% done | ETA(epoch): 398s + [Epoch 16][180/200] loss=0.691813 avg=0.663843 VRAM=38.5GiB | 3.4% done | ETA(epoch): 265s + [Epoch 16][190/200] loss=0.628005 avg=0.664305 VRAM=38.6GiB | 3.4% done | ETA(epoch): 133s + [Epoch 16][200/200] loss=0.531585 avg=0.662665 VRAM=38.5GiB | 3.4% done | ETA(epoch): 0s + Train loss: 0.662665 (2652.0s) ETA: 24284min + Val loss: 0.713610 [t_0.0-0.2=0.9522 t_0.2-0.4=0.7566 t_0.4-0.6=0.6510 t_0.6-0.8=0.5789 t_0.8-1.0=0.6102] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 17/499 (3% done) --- + [Epoch 17][10/200] loss=0.532787 avg=0.645457 VRAM=38.6GiB | 3.4% done | ETA(epoch): 2516s + [Epoch 17][20/200] loss=0.558520 avg=0.652807 VRAM=38.5GiB | 3.4% done | ETA(epoch): 2386s + [Epoch 17][30/200] loss=0.524139 avg=0.653429 VRAM=38.6GiB | 3.4% done | ETA(epoch): 2253s + [Epoch 17][40/200] loss=0.708593 avg=0.656417 VRAM=38.5GiB | 3.4% done | ETA(epoch): 2121s + [Epoch 17][50/200] loss=0.690084 avg=0.665052 VRAM=38.6GiB | 3.5% done | ETA(epoch): 1988s + [Epoch 17][60/200] loss=0.693160 avg=0.667795 VRAM=38.5GiB | 3.5% done | ETA(epoch): 1855s + [Epoch 17][70/200] loss=0.692755 avg=0.670948 VRAM=38.6GiB | 3.5% done | ETA(epoch): 1723s + [Epoch 17][80/200] loss=0.638600 avg=0.673826 VRAM=38.5GiB | 3.5% done | ETA(epoch): 1591s + [Epoch 17][90/200] loss=0.720722 avg=0.672318 VRAM=38.6GiB | 3.5% done | ETA(epoch): 1461s + [Epoch 17][100/200] loss=0.681933 avg=0.674986 VRAM=38.5GiB | 3.5% done | ETA(epoch): 1328s + [Epoch 17][110/200] loss=0.705993 avg=0.676720 VRAM=38.6GiB | 3.5% done | ETA(epoch): 1195s + [Epoch 17][120/200] loss=0.667393 avg=0.675971 VRAM=38.5GiB | 3.5% done | ETA(epoch): 1062s + [Epoch 17][130/200] loss=0.607093 avg=0.677096 VRAM=38.6GiB | 3.5% done | ETA(epoch): 929s + [Epoch 17][140/200] loss=0.746451 avg=0.679244 VRAM=38.5GiB | 3.5% done | ETA(epoch): 797s + [Epoch 17][150/200] loss=0.677549 avg=0.678838 VRAM=38.6GiB | 3.5% done | ETA(epoch): 664s + [Epoch 17][160/200] loss=0.629835 avg=0.678449 VRAM=38.5GiB | 3.6% done | ETA(epoch): 531s + [Epoch 17][170/200] loss=0.783914 avg=0.679531 VRAM=38.6GiB | 3.6% done | ETA(epoch): 398s + [Epoch 17][180/200] loss=0.725991 avg=0.680207 VRAM=38.5GiB | 3.6% done | ETA(epoch): 265s + [Epoch 17][190/200] loss=0.566970 avg=0.678552 VRAM=38.6GiB | 3.6% done | ETA(epoch): 133s + [Epoch 17][200/200] loss=0.723665 avg=0.678787 VRAM=38.5GiB | 3.6% done | ETA(epoch): 0s + Train loss: 0.678787 (2654.3s) ETA: 24186min + Val loss: 0.699023 [t_0.0-0.2=0.9363 t_0.2-0.4=0.7530 t_0.4-0.6=0.6476 t_0.6-0.8=0.5651 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0017 +[MEM @ epoch 17 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 18/499 (4% done) --- + [Epoch 18][10/200] loss=0.662185 avg=0.624472 VRAM=38.6GiB | 3.6% done | ETA(epoch): 2518s + [Epoch 18][20/200] loss=0.592822 avg=0.647468 VRAM=38.5GiB | 3.6% done | ETA(epoch): 2390s + [Epoch 18][30/200] loss=0.606238 avg=0.645788 VRAM=38.6GiB | 3.6% done | ETA(epoch): 2255s + [Epoch 18][40/200] loss=0.582547 avg=0.648922 VRAM=38.5GiB | 3.6% done | ETA(epoch): 2123s + [Epoch 18][50/200] loss=0.618095 avg=0.650560 VRAM=38.6GiB | 3.6% done | ETA(epoch): 1990s + [Epoch 18][60/200] loss=0.604248 avg=0.652162 VRAM=38.5GiB | 3.7% done | ETA(epoch): 1858s + [Epoch 18][70/200] loss=0.632262 avg=0.657201 VRAM=38.6GiB | 3.7% done | ETA(epoch): 1724s + [Epoch 18][80/200] loss=0.737843 avg=0.662696 VRAM=38.5GiB | 3.7% done | ETA(epoch): 1592s + [Epoch 18][90/200] loss=0.663906 avg=0.659954 VRAM=38.6GiB | 3.7% done | ETA(epoch): 1459s + [Epoch 18][100/200] loss=0.697132 avg=0.663411 VRAM=38.5GiB | 3.7% done | ETA(epoch): 1326s + [Epoch 18][110/200] loss=0.687864 avg=0.663832 VRAM=38.6GiB | 3.7% done | ETA(epoch): 1193s + [Epoch 18][120/200] loss=0.661666 avg=0.665950 VRAM=38.5GiB | 3.7% done | ETA(epoch): 1061s + [Epoch 18][130/200] loss=0.609358 avg=0.664331 VRAM=38.6GiB | 3.7% done | ETA(epoch): 928s + [Epoch 18][140/200] loss=0.565160 avg=0.662700 VRAM=38.5GiB | 3.7% done | ETA(epoch): 797s + [Epoch 18][150/200] loss=0.735364 avg=0.663500 VRAM=38.6GiB | 3.8% done | ETA(epoch): 664s + [Epoch 18][160/200] loss=0.732921 avg=0.663778 VRAM=38.5GiB | 3.8% done | ETA(epoch): 531s + [Epoch 18][170/200] loss=0.541496 avg=0.661057 VRAM=38.6GiB | 3.8% done | ETA(epoch): 398s + [Epoch 18][180/200] loss=0.635201 avg=0.660449 VRAM=38.5GiB | 3.8% done | ETA(epoch): 266s + [Epoch 18][190/200] loss=0.507739 avg=0.658320 VRAM=38.6GiB | 3.8% done | ETA(epoch): 133s + [Epoch 18][200/200] loss=0.651155 avg=0.659603 VRAM=38.5GiB | 3.8% done | ETA(epoch): 0s + Train loss: 0.659603 (2655.7s) ETA: 24091min + Val loss: 0.710886 [t_0.0-0.2=0.9552 t_0.2-0.4=0.7572 t_0.4-0.6=0.6551 t_0.6-0.8=0.5815 t_0.8-1.0=0.6105] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 20.1/188.5 GiB (10.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 19/499 (4% done) --- + [Epoch 19][10/200] loss=0.621432 avg=0.650035 VRAM=38.6GiB | 3.8% done | ETA(epoch): 2581s + [Epoch 19][20/200] loss=0.632699 avg=0.637670 VRAM=38.5GiB | 3.8% done | ETA(epoch): 2414s + [Epoch 19][30/200] loss=0.608223 avg=0.651277 VRAM=38.6GiB | 3.8% done | ETA(epoch): 2271s + [Epoch 19][40/200] loss=0.788440 avg=0.658672 VRAM=38.5GiB | 3.8% done | ETA(epoch): 2133s + [Epoch 19][50/200] loss=0.649912 avg=0.657359 VRAM=38.6GiB | 3.9% done | ETA(epoch): 1998s + [Epoch 19][60/200] loss=0.732186 avg=0.663367 VRAM=38.5GiB | 3.9% done | ETA(epoch): 1863s + [Epoch 19][70/200] loss=0.665242 avg=0.671818 VRAM=38.6GiB | 3.9% done | ETA(epoch): 1730s + [Epoch 19][80/200] loss=0.759042 avg=0.675567 VRAM=38.5GiB | 3.9% done | ETA(epoch): 1595s + [Epoch 19][90/200] loss=0.712732 avg=0.678865 VRAM=38.6GiB | 3.9% done | ETA(epoch): 1463s + [Epoch 19][100/200] loss=0.655438 avg=0.675572 VRAM=38.5GiB | 3.9% done | ETA(epoch): 1331s + [Epoch 19][110/200] loss=0.622969 avg=0.677616 VRAM=38.6GiB | 3.9% done | ETA(epoch): 1198s + [Epoch 19][120/200] loss=0.820972 avg=0.676993 VRAM=38.5GiB | 3.9% done | ETA(epoch): 1064s + [Epoch 19][130/200] loss=0.734433 avg=0.675721 VRAM=38.6GiB | 3.9% done | ETA(epoch): 931s + [Epoch 19][140/200] loss=0.658716 avg=0.674780 VRAM=38.5GiB | 3.9% done | ETA(epoch): 798s + [Epoch 19][150/200] loss=0.681685 avg=0.674693 VRAM=38.6GiB | 4.0% done | ETA(epoch): 665s + [Epoch 19][160/200] loss=0.629254 avg=0.676581 VRAM=38.5GiB | 4.0% done | ETA(epoch): 532s + [Epoch 19][170/200] loss=0.657480 avg=0.677444 VRAM=38.6GiB | 4.0% done | ETA(epoch): 399s + [Epoch 19][180/200] loss=0.662136 avg=0.676983 VRAM=38.5GiB | 4.0% done | ETA(epoch): 266s + [Epoch 19][190/200] loss=0.731880 avg=0.677151 VRAM=38.6GiB | 4.0% done | ETA(epoch): 133s + [Epoch 19][200/200] loss=0.630842 avg=0.677270 VRAM=38.5GiB | 4.0% done | ETA(epoch): 0s + Train loss: 0.677270 (2659.7s) ETA: 24003min + Val loss: 0.710663 [t_0.0-0.2=0.9344 t_0.2-0.4=0.7543 t_0.4-0.6=0.6571 t_0.6-0.8=0.5874 t_0.8-1.0=0.6055] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 20/499 (4% done) --- + [Epoch 20][10/200] loss=0.671028 avg=0.700224 VRAM=38.6GiB | 4.0% done | ETA(epoch): 2518s + [Epoch 20][20/200] loss=0.702472 avg=0.688254 VRAM=38.5GiB | 4.0% done | ETA(epoch): 2389s + [Epoch 20][30/200] loss=0.651542 avg=0.683407 VRAM=38.6GiB | 4.0% done | ETA(epoch): 2254s + [Epoch 20][40/200] loss=0.756326 avg=0.677083 VRAM=38.5GiB | 4.0% done | ETA(epoch): 2122s + [Epoch 20][50/200] loss=0.720403 avg=0.678036 VRAM=38.6GiB | 4.0% done | ETA(epoch): 1990s + [Epoch 20][60/200] loss=0.657395 avg=0.675246 VRAM=38.5GiB | 4.1% done | ETA(epoch): 1857s + [Epoch 20][70/200] loss=0.571319 avg=0.674921 VRAM=38.6GiB | 4.1% done | ETA(epoch): 1729s + [Epoch 20][80/200] loss=0.627191 avg=0.670054 VRAM=38.5GiB | 4.1% done | ETA(epoch): 1596s + [Epoch 20][90/200] loss=0.615053 avg=0.672052 VRAM=38.6GiB | 4.1% done | ETA(epoch): 1462s + [Epoch 20][100/200] loss=0.672079 avg=0.669144 VRAM=38.5GiB | 4.1% done | ETA(epoch): 1329s + [Epoch 20][110/200] loss=0.731587 avg=0.669940 VRAM=38.6GiB | 4.1% done | ETA(epoch): 1196s + [Epoch 20][120/200] loss=0.633509 avg=0.666254 VRAM=38.5GiB | 4.1% done | ETA(epoch): 1063s + [Epoch 20][130/200] loss=0.704164 avg=0.669657 VRAM=38.6GiB | 4.1% done | ETA(epoch): 930s + [Epoch 20][140/200] loss=0.630176 avg=0.667819 VRAM=38.5GiB | 4.1% done | ETA(epoch): 797s + [Epoch 20][150/200] loss=0.769578 avg=0.668353 VRAM=38.6GiB | 4.2% done | ETA(epoch): 664s + [Epoch 20][160/200] loss=0.688145 avg=0.668416 VRAM=38.5GiB | 4.2% done | ETA(epoch): 531s + [Epoch 20][170/200] loss=0.605232 avg=0.669098 VRAM=38.6GiB | 4.2% done | ETA(epoch): 398s + [Epoch 20][180/200] loss=0.756647 avg=0.667334 VRAM=38.5GiB | 4.2% done | ETA(epoch): 265s + [Epoch 20][190/200] loss=0.766264 avg=0.667863 VRAM=38.6GiB | 4.2% done | ETA(epoch): 133s + [Epoch 20][200/200] loss=0.681427 avg=0.667906 VRAM=38.5GiB | 4.2% done | ETA(epoch): 0s + Train loss: 0.667906 (2654.4s) ETA: 23916min + Val loss: 0.693509 [t_0.0-0.2=0.9601 t_0.2-0.4=0.7673 t_0.4-0.6=0.6229 t_0.6-0.8=0.5838 t_0.8-1.0=0.5928] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 21/499 (4% done) --- + [Epoch 21][10/200] loss=0.618304 avg=0.648388 VRAM=38.6GiB | 4.2% done | ETA(epoch): 2514s + [Epoch 21][20/200] loss=0.787410 avg=0.671022 VRAM=38.5GiB | 4.2% done | ETA(epoch): 2383s + [Epoch 21][30/200] loss=0.660969 avg=0.673513 VRAM=38.6GiB | 4.2% done | ETA(epoch): 2254s + [Epoch 21][40/200] loss=0.704788 avg=0.673311 VRAM=38.5GiB | 4.2% done | ETA(epoch): 2121s + [Epoch 21][50/200] loss=0.786350 avg=0.677013 VRAM=38.6GiB | 4.2% done | ETA(epoch): 1989s + [Epoch 21][60/200] loss=0.633231 avg=0.679166 VRAM=38.5GiB | 4.3% done | ETA(epoch): 1856s + [Epoch 21][70/200] loss=0.641178 avg=0.678021 VRAM=38.6GiB | 4.3% done | ETA(epoch): 1724s + [Epoch 21][80/200] loss=0.769357 avg=0.673566 VRAM=38.5GiB | 4.3% done | ETA(epoch): 1591s + [Epoch 21][90/200] loss=0.690740 avg=0.673400 VRAM=38.6GiB | 4.3% done | ETA(epoch): 1459s + [Epoch 21][100/200] loss=0.740039 avg=0.673716 VRAM=38.5GiB | 4.3% done | ETA(epoch): 1326s + [Epoch 21][110/200] loss=0.610724 avg=0.671884 VRAM=38.6GiB | 4.3% done | ETA(epoch): 1193s + [Epoch 21][120/200] loss=0.787936 avg=0.673876 VRAM=38.5GiB | 4.3% done | ETA(epoch): 1061s + [Epoch 21][130/200] loss=0.565502 avg=0.675994 VRAM=38.6GiB | 4.3% done | ETA(epoch): 928s + [Epoch 21][140/200] loss=0.686631 avg=0.676221 VRAM=38.5GiB | 4.3% done | ETA(epoch): 795s + [Epoch 21][150/200] loss=0.678646 avg=0.672900 VRAM=38.6GiB | 4.3% done | ETA(epoch): 663s + [Epoch 21][160/200] loss=0.675983 avg=0.674330 VRAM=38.5GiB | 4.4% done | ETA(epoch): 530s + [Epoch 21][170/200] loss=0.615080 avg=0.674455 VRAM=38.6GiB | 4.4% done | ETA(epoch): 398s + [Epoch 21][180/200] loss=0.722734 avg=0.672832 VRAM=38.5GiB | 4.4% done | ETA(epoch): 265s + [Epoch 21][190/200] loss=0.565575 avg=0.672321 VRAM=38.6GiB | 4.4% done | ETA(epoch): 133s + [Epoch 21][200/200] loss=0.587597 avg=0.672615 VRAM=38.5GiB | 4.4% done | ETA(epoch): 0s + Train loss: 0.672615 (2651.0s) ETA: 23831min + Val loss: 0.733319 [t_0.0-0.2=0.9276 t_0.2-0.4=0.7663 t_0.4-0.6=0.6624 t_0.6-0.8=0.5938 t_0.8-1.0=0.6120] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 22/499 (4% done) --- + [Epoch 22][10/200] loss=0.735978 avg=0.677297 VRAM=38.6GiB | 4.4% done | ETA(epoch): 2514s + [Epoch 22][20/200] loss=0.615181 avg=0.670414 VRAM=38.5GiB | 4.4% done | ETA(epoch): 2382s + [Epoch 22][30/200] loss=0.631468 avg=0.680977 VRAM=38.6GiB | 4.4% done | ETA(epoch): 2252s + [Epoch 22][40/200] loss=0.772997 avg=0.673109 VRAM=38.5GiB | 4.4% done | ETA(epoch): 2120s + [Epoch 22][50/200] loss=0.596837 avg=0.669061 VRAM=38.6GiB | 4.5% done | ETA(epoch): 1987s + [Epoch 22][60/200] loss=0.723465 avg=0.669585 VRAM=38.5GiB | 4.5% done | ETA(epoch): 1855s + [Epoch 22][70/200] loss=0.619672 avg=0.671149 VRAM=38.6GiB | 4.5% done | ETA(epoch): 1722s + [Epoch 22][80/200] loss=0.633181 avg=0.669529 VRAM=38.5GiB | 4.5% done | ETA(epoch): 1591s + [Epoch 22][90/200] loss=0.593419 avg=0.666692 VRAM=38.6GiB | 4.5% done | ETA(epoch): 1458s + [Epoch 22][100/200] loss=0.604044 avg=0.661853 VRAM=38.5GiB | 4.5% done | ETA(epoch): 1326s + [Epoch 22][110/200] loss=0.888763 avg=0.663623 VRAM=38.6GiB | 4.5% done | ETA(epoch): 1193s + [Epoch 22][120/200] loss=0.638049 avg=0.663262 VRAM=38.5GiB | 4.5% done | ETA(epoch): 1060s + [Epoch 22][130/200] loss=0.595551 avg=0.665977 VRAM=38.6GiB | 4.5% done | ETA(epoch): 928s + [Epoch 22][140/200] loss=0.617929 avg=0.667782 VRAM=38.5GiB | 4.5% done | ETA(epoch): 795s + [Epoch 22][150/200] loss=0.930043 avg=0.670434 VRAM=38.6GiB | 4.5% done | ETA(epoch): 663s + [Epoch 22][160/200] loss=0.621603 avg=0.668659 VRAM=38.5GiB | 4.6% done | ETA(epoch): 530s + [Epoch 22][170/200] loss=0.645267 avg=0.667918 VRAM=38.6GiB | 4.6% done | ETA(epoch): 398s + [Epoch 22][180/200] loss=0.647117 avg=0.666471 VRAM=38.5GiB | 4.6% done | ETA(epoch): 265s + [Epoch 22][190/200] loss=0.551166 avg=0.666400 VRAM=38.6GiB | 4.6% done | ETA(epoch): 133s + [Epoch 22][200/200] loss=0.846760 avg=0.667901 VRAM=38.5GiB | 4.6% done | ETA(epoch): 0s + Train loss: 0.667901 (2650.9s) ETA: 23750min + Val loss: 0.703443 [t_0.0-0.2=0.9377 t_0.2-0.4=0.7594 t_0.4-0.6=0.6486 t_0.6-0.8=0.5640 t_0.8-1.0=0.6117] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 23/499 (5% done) --- + [Epoch 23][10/200] loss=0.649008 avg=0.656223 VRAM=38.6GiB | 4.6% done | ETA(epoch): 2522s + [Epoch 23][20/200] loss=0.657953 avg=0.647813 VRAM=38.5GiB | 4.6% done | ETA(epoch): 2387s + [Epoch 23][30/200] loss=0.616934 avg=0.649583 VRAM=38.6GiB | 4.6% done | ETA(epoch): 2255s + [Epoch 23][40/200] loss=0.636120 avg=0.652672 VRAM=38.5GiB | 4.6% done | ETA(epoch): 2121s + [Epoch 23][50/200] loss=0.818390 avg=0.655722 VRAM=38.6GiB | 4.7% done | ETA(epoch): 1997s + [Epoch 23][60/200] loss=0.713088 avg=0.659948 VRAM=38.5GiB | 4.7% done | ETA(epoch): 1862s + [Epoch 23][70/200] loss=0.627379 avg=0.660885 VRAM=38.6GiB | 4.7% done | ETA(epoch): 1729s + [Epoch 23][80/200] loss=0.604940 avg=0.660164 VRAM=38.5GiB | 4.7% done | ETA(epoch): 1595s + [Epoch 23][90/200] loss=0.707308 avg=0.661705 VRAM=38.6GiB | 4.7% done | ETA(epoch): 1462s + [Epoch 23][100/200] loss=0.697382 avg=0.663707 VRAM=38.5GiB | 4.7% done | ETA(epoch): 1329s + [Epoch 23][110/200] loss=0.742729 avg=0.665565 VRAM=38.6GiB | 4.7% done | ETA(epoch): 1196s + [Epoch 23][120/200] loss=0.641742 avg=0.664310 VRAM=38.5GiB | 4.7% done | ETA(epoch): 1063s + [Epoch 23][130/200] loss=0.541165 avg=0.664369 VRAM=38.6GiB | 4.7% done | ETA(epoch): 930s + [Epoch 23][140/200] loss=0.646360 avg=0.664843 VRAM=38.5GiB | 4.7% done | ETA(epoch): 797s + [Epoch 23][150/200] loss=0.687587 avg=0.664233 VRAM=38.6GiB | 4.8% done | ETA(epoch): 664s + [Epoch 23][160/200] loss=0.621875 avg=0.664485 VRAM=38.5GiB | 4.8% done | ETA(epoch): 531s + [Epoch 23][170/200] loss=0.743926 avg=0.665094 VRAM=38.6GiB | 4.8% done | ETA(epoch): 398s + [Epoch 23][180/200] loss=0.619605 avg=0.665049 VRAM=38.5GiB | 4.8% done | ETA(epoch): 266s + [Epoch 23][190/200] loss=0.561047 avg=0.664032 VRAM=38.6GiB | 4.8% done | ETA(epoch): 133s + [Epoch 23][200/200] loss=0.735700 avg=0.664382 VRAM=38.5GiB | 4.8% done | ETA(epoch): 0s + Train loss: 0.664382 (2655.1s) ETA: 23672min + Val loss: 0.707459 [t_0.0-0.2=0.9140 t_0.2-0.4=0.7587 t_0.4-0.6=0.6664 t_0.6-0.8=0.5754 t_0.8-1.0=0.6049] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 24/499 (5% done) --- + [Epoch 24][10/200] loss=0.509686 avg=0.597763 VRAM=38.6GiB | 4.8% done | ETA(epoch): 2521s + [Epoch 24][20/200] loss=0.599379 avg=0.626935 VRAM=38.5GiB | 4.8% done | ETA(epoch): 2387s + [Epoch 24][30/200] loss=0.561116 avg=0.641098 VRAM=38.6GiB | 4.8% done | ETA(epoch): 2255s + [Epoch 24][40/200] loss=0.600483 avg=0.637809 VRAM=38.5GiB | 4.8% done | ETA(epoch): 2122s + [Epoch 24][50/200] loss=0.685030 avg=0.643036 VRAM=38.6GiB | 4.9% done | ETA(epoch): 1989s + [Epoch 24][60/200] loss=0.588948 avg=0.640265 VRAM=38.5GiB | 4.9% done | ETA(epoch): 1857s + [Epoch 24][70/200] loss=0.597689 avg=0.638694 VRAM=38.6GiB | 4.9% done | ETA(epoch): 1724s + [Epoch 24][80/200] loss=0.743186 avg=0.644217 VRAM=38.5GiB | 4.9% done | ETA(epoch): 1591s + [Epoch 24][90/200] loss=0.604755 avg=0.643452 VRAM=38.6GiB | 4.9% done | ETA(epoch): 1459s + [Epoch 24][100/200] loss=0.693258 avg=0.643181 VRAM=38.5GiB | 4.9% done | ETA(epoch): 1328s + [Epoch 24][110/200] loss=0.682112 avg=0.649111 VRAM=38.6GiB | 4.9% done | ETA(epoch): 1195s + [Epoch 24][120/200] loss=0.672561 avg=0.652221 VRAM=38.5GiB | 4.9% done | ETA(epoch): 1062s + [Epoch 24][130/200] loss=0.718438 avg=0.655140 VRAM=38.6GiB | 4.9% done | ETA(epoch): 930s + [Epoch 24][140/200] loss=0.716447 avg=0.655651 VRAM=38.5GiB | 4.9% done | ETA(epoch): 797s + [Epoch 24][150/200] loss=0.732560 avg=0.658283 VRAM=38.6GiB | 5.0% done | ETA(epoch): 664s + [Epoch 24][160/200] loss=0.716793 avg=0.658791 VRAM=38.5GiB | 5.0% done | ETA(epoch): 531s + [Epoch 24][170/200] loss=0.645130 avg=0.659156 VRAM=38.6GiB | 5.0% done | ETA(epoch): 398s + [Epoch 24][180/200] loss=0.609708 avg=0.657611 VRAM=38.5GiB | 5.0% done | ETA(epoch): 265s + [Epoch 24][190/200] loss=0.527686 avg=0.657001 VRAM=38.6GiB | 5.0% done | ETA(epoch): 133s + [Epoch 24][200/200] loss=0.666183 avg=0.656367 VRAM=38.5GiB | 5.0% done | ETA(epoch): 0s + Train loss: 0.656367 (2658.7s) ETA: 23599min + Val loss: 0.706721 [t_0.0-0.2=0.9394 t_0.2-0.4=0.7527 t_0.4-0.6=0.6426 t_0.6-0.8=0.5718 t_0.8-1.0=0.5927] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 25/499 (5% done) --- + [Epoch 25][10/200] loss=0.725428 avg=0.653745 VRAM=38.6GiB | 5.0% done | ETA(epoch): 2518s + [Epoch 25][20/200] loss=0.589848 avg=0.651473 VRAM=38.5GiB | 5.0% done | ETA(epoch): 2386s + [Epoch 25][30/200] loss=0.636858 avg=0.652437 VRAM=38.6GiB | 5.0% done | ETA(epoch): 2255s + [Epoch 25][40/200] loss=0.671089 avg=0.654762 VRAM=38.5GiB | 5.0% done | ETA(epoch): 2122s + [Epoch 25][50/200] loss=0.633870 avg=0.666549 VRAM=38.6GiB | 5.1% done | ETA(epoch): 1989s + [Epoch 25][60/200] loss=0.599631 avg=0.664171 VRAM=38.5GiB | 5.1% done | ETA(epoch): 1858s + [Epoch 25][70/200] loss=0.560072 avg=0.659969 VRAM=38.6GiB | 5.1% done | ETA(epoch): 1725s + [Epoch 25][80/200] loss=0.787037 avg=0.665039 VRAM=38.5GiB | 5.1% done | ETA(epoch): 1592s + [Epoch 25][90/200] loss=0.757037 avg=0.665223 VRAM=38.6GiB | 5.1% done | ETA(epoch): 1459s + [Epoch 25][100/200] loss=0.602030 avg=0.664045 VRAM=38.5GiB | 5.1% done | ETA(epoch): 1326s + [Epoch 25][110/200] loss=0.703772 avg=0.663387 VRAM=38.6GiB | 5.1% done | ETA(epoch): 1194s + [Epoch 25][120/200] loss=0.652906 avg=0.661489 VRAM=38.5GiB | 5.1% done | ETA(epoch): 1061s + [Epoch 25][130/200] loss=0.697359 avg=0.663103 VRAM=38.6GiB | 5.1% done | ETA(epoch): 929s + [Epoch 25][140/200] loss=0.769796 avg=0.661553 VRAM=38.5GiB | 5.1% done | ETA(epoch): 796s + [Epoch 25][150/200] loss=0.783244 avg=0.666009 VRAM=38.6GiB | 5.1% done | ETA(epoch): 664s + [Epoch 25][160/200] loss=0.616739 avg=0.663836 VRAM=38.5GiB | 5.2% done | ETA(epoch): 531s + [Epoch 25][170/200] loss=0.771428 avg=0.664466 VRAM=38.6GiB | 5.2% done | ETA(epoch): 398s + [Epoch 25][180/200] loss=0.670853 avg=0.667778 VRAM=38.5GiB | 5.2% done | ETA(epoch): 266s + [Epoch 25][190/200] loss=0.650928 avg=0.667949 VRAM=38.6GiB | 5.2% done | ETA(epoch): 133s + [Epoch 25][200/200] loss=0.732114 avg=0.669755 VRAM=38.5GiB | 5.2% done | ETA(epoch): 0s + Train loss: 0.669755 (2655.7s) ETA: 23526min + Val loss: 0.717998 [t_0.0-0.2=0.9510 t_0.2-0.4=0.7713 t_0.4-0.6=0.6463 t_0.6-0.8=0.5695 t_0.8-1.0=0.6139] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 26/499 (5% done) --- + [Epoch 26][10/200] loss=0.649350 avg=0.692824 VRAM=38.6GiB | 5.2% done | ETA(epoch): 2519s + [Epoch 26][20/200] loss=0.764779 avg=0.703007 VRAM=38.5GiB | 5.2% done | ETA(epoch): 2388s + [Epoch 26][30/200] loss=0.664389 avg=0.694508 VRAM=38.6GiB | 5.2% done | ETA(epoch): 2254s + [Epoch 26][40/200] loss=0.668753 avg=0.688975 VRAM=38.5GiB | 5.2% done | ETA(epoch): 2123s + [Epoch 26][50/200] loss=0.634587 avg=0.686680 VRAM=38.6GiB | 5.2% done | ETA(epoch): 1990s + [Epoch 26][60/200] loss=0.637295 avg=0.681446 VRAM=38.5GiB | 5.3% done | ETA(epoch): 1857s + [Epoch 26][70/200] loss=0.802439 avg=0.680321 VRAM=38.6GiB | 5.3% done | ETA(epoch): 1724s + [Epoch 26][80/200] loss=0.639768 avg=0.679190 VRAM=38.5GiB | 5.3% done | ETA(epoch): 1592s + [Epoch 26][90/200] loss=0.833213 avg=0.679654 VRAM=38.6GiB | 5.3% done | ETA(epoch): 1459s + [Epoch 26][100/200] loss=0.755923 avg=0.682296 VRAM=38.5GiB | 5.3% done | ETA(epoch): 1326s + [Epoch 26][110/200] loss=0.581707 avg=0.680829 VRAM=38.6GiB | 5.3% done | ETA(epoch): 1194s + [Epoch 26][120/200] loss=0.646411 avg=0.678326 VRAM=38.5GiB | 5.3% done | ETA(epoch): 1064s + [Epoch 26][130/200] loss=0.651137 avg=0.678313 VRAM=38.6GiB | 5.3% done | ETA(epoch): 931s + [Epoch 26][140/200] loss=0.712073 avg=0.675174 VRAM=38.5GiB | 5.3% done | ETA(epoch): 798s + [Epoch 26][150/200] loss=0.665344 avg=0.676841 VRAM=38.6GiB | 5.3% done | ETA(epoch): 665s + [Epoch 26][160/200] loss=0.762071 avg=0.676354 VRAM=38.5GiB | 5.4% done | ETA(epoch): 532s + [Epoch 26][170/200] loss=0.630278 avg=0.675084 VRAM=38.6GiB | 5.4% done | ETA(epoch): 399s + [Epoch 26][180/200] loss=0.667519 avg=0.675649 VRAM=38.5GiB | 5.4% done | ETA(epoch): 266s + [Epoch 26][190/200] loss=0.686784 avg=0.675114 VRAM=38.6GiB | 5.4% done | ETA(epoch): 133s + [Epoch 26][200/200] loss=0.610020 avg=0.674961 VRAM=38.5GiB | 5.4% done | ETA(epoch): 0s + Train loss: 0.674961 (2657.2s) ETA: 23455min + Val loss: 0.711353 [t_0.0-0.2=0.9421 t_0.2-0.4=0.7626 t_0.4-0.6=0.6543 t_0.6-0.8=0.5665 t_0.8-1.0=0.5971] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 21.0/188.5 GiB (11.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 27/499 (5% done) --- + [Epoch 27][10/200] loss=0.708360 avg=0.721144 VRAM=38.6GiB | 5.4% done | ETA(epoch): 2517s + [Epoch 27][20/200] loss=0.794358 avg=0.709437 VRAM=38.5GiB | 5.4% done | ETA(epoch): 2384s + [Epoch 27][30/200] loss=0.586328 avg=0.694423 VRAM=38.6GiB | 5.4% done | ETA(epoch): 2254s + [Epoch 27][40/200] loss=0.588966 avg=0.696071 VRAM=38.5GiB | 5.4% done | ETA(epoch): 2120s + [Epoch 27][50/200] loss=0.588233 avg=0.690702 VRAM=38.6GiB | 5.5% done | ETA(epoch): 1988s + [Epoch 27][60/200] loss=0.690657 avg=0.689975 VRAM=38.5GiB | 5.5% done | ETA(epoch): 1855s + [Epoch 27][70/200] loss=0.714431 avg=0.692010 VRAM=38.6GiB | 5.5% done | ETA(epoch): 1723s + [Epoch 27][80/200] loss=0.706632 avg=0.686179 VRAM=38.5GiB | 5.5% done | ETA(epoch): 1595s + [Epoch 27][90/200] loss=0.631152 avg=0.688056 VRAM=38.6GiB | 5.5% done | ETA(epoch): 1462s + [Epoch 27][100/200] loss=0.642458 avg=0.686921 VRAM=38.5GiB | 5.5% done | ETA(epoch): 1329s + [Epoch 27][110/200] loss=0.602943 avg=0.685747 VRAM=38.6GiB | 5.5% done | ETA(epoch): 1196s + [Epoch 27][120/200] loss=0.591013 avg=0.683238 VRAM=38.5GiB | 5.5% done | ETA(epoch): 1063s + [Epoch 27][130/200] loss=0.615410 avg=0.681430 VRAM=38.6GiB | 5.5% done | ETA(epoch): 930s + [Epoch 27][140/200] loss=0.638523 avg=0.678634 VRAM=38.5GiB | 5.5% done | ETA(epoch): 797s + [Epoch 27][150/200] loss=0.757222 avg=0.678294 VRAM=38.6GiB | 5.5% done | ETA(epoch): 664s + [Epoch 27][160/200] loss=0.587245 avg=0.675295 VRAM=38.5GiB | 5.6% done | ETA(epoch): 531s + [Epoch 27][170/200] loss=0.814413 avg=0.676104 VRAM=38.6GiB | 5.6% done | ETA(epoch): 398s + [Epoch 27][180/200] loss=0.618194 avg=0.675940 VRAM=38.5GiB | 5.6% done | ETA(epoch): 266s + [Epoch 27][190/200] loss=0.780521 avg=0.675392 VRAM=38.6GiB | 5.6% done | ETA(epoch): 133s + [Epoch 27][200/200] loss=0.591762 avg=0.675554 VRAM=38.5GiB | 5.6% done | ETA(epoch): 0s + Train loss: 0.675554 (2655.5s) ETA: 23385min + Val loss: 0.704827 [t_0.0-0.2=0.9506 t_0.2-0.4=0.7596 t_0.4-0.6=0.6312 t_0.6-0.8=0.5741 t_0.8-1.0=0.5932] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 21.1/188.5 GiB (11.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 28/499 (6% done) --- + [Epoch 28][10/200] loss=0.718712 avg=0.648283 VRAM=38.6GiB | 5.6% done | ETA(epoch): 2521s + [Epoch 28][20/200] loss=0.809738 avg=0.668182 VRAM=38.5GiB | 5.6% done | ETA(epoch): 2388s + [Epoch 28][30/200] loss=0.672952 avg=0.669004 VRAM=38.6GiB | 5.6% done | ETA(epoch): 2255s + [Epoch 28][40/200] loss=0.649355 avg=0.672839 VRAM=38.5GiB | 5.6% done | ETA(epoch): 2123s + [Epoch 28][50/200] loss=0.706804 avg=0.672950 VRAM=38.6GiB | 5.7% done | ETA(epoch): 1990s + [Epoch 28][60/200] loss=0.729768 avg=0.673176 VRAM=38.5GiB | 5.7% done | ETA(epoch): 1858s + [Epoch 28][70/200] loss=0.633482 avg=0.678995 VRAM=38.6GiB | 5.7% done | ETA(epoch): 1725s + [Epoch 28][80/200] loss=0.676543 avg=0.682693 VRAM=38.5GiB | 5.7% done | ETA(epoch): 1592s + [Epoch 28][90/200] loss=0.761550 avg=0.678864 VRAM=38.6GiB | 5.7% done | ETA(epoch): 1460s + [Epoch 28][100/200] loss=0.629471 avg=0.676950 VRAM=38.5GiB | 5.7% done | ETA(epoch): 1327s + [Epoch 28][110/200] loss=0.642205 avg=0.673759 VRAM=38.6GiB | 5.7% done | ETA(epoch): 1194s + [Epoch 28][120/200] loss=0.786939 avg=0.674135 VRAM=38.5GiB | 5.7% done | ETA(epoch): 1061s + [Epoch 28][130/200] loss=0.577093 avg=0.673719 VRAM=38.6GiB | 5.7% done | ETA(epoch): 929s + [Epoch 28][140/200] loss=0.608537 avg=0.672904 VRAM=38.5GiB | 5.7% done | ETA(epoch): 796s + [Epoch 28][150/200] loss=0.693155 avg=0.672032 VRAM=38.6GiB | 5.8% done | ETA(epoch): 664s + [Epoch 28][160/200] loss=0.684808 avg=0.671080 VRAM=38.5GiB | 5.8% done | ETA(epoch): 531s + [Epoch 28][170/200] loss=0.666378 avg=0.669242 VRAM=38.6GiB | 5.8% done | ETA(epoch): 398s + [Epoch 28][180/200] loss=0.692868 avg=0.669086 VRAM=38.5GiB | 5.8% done | ETA(epoch): 265s + [Epoch 28][190/200] loss=0.597485 avg=0.669661 VRAM=38.6GiB | 5.8% done | ETA(epoch): 133s + [Epoch 28][200/200] loss=0.585752 avg=0.668017 VRAM=38.5GiB | 5.8% done | ETA(epoch): 0s + Train loss: 0.668017 (2653.9s) ETA: 23316min + Val loss: 0.714314 [t_0.0-0.2=0.9407 t_0.2-0.4=0.7490 t_0.4-0.6=0.6462 t_0.6-0.8=0.5656 t_0.8-1.0=0.6030] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 29/499 (6% done) --- + [Epoch 29][10/200] loss=0.693937 avg=0.665791 VRAM=38.6GiB | 5.8% done | ETA(epoch): 2519s + [Epoch 29][20/200] loss=0.648496 avg=0.673706 VRAM=38.5GiB | 5.8% done | ETA(epoch): 2389s + [Epoch 29][30/200] loss=0.775117 avg=0.669578 VRAM=38.6GiB | 5.8% done | ETA(epoch): 2256s + [Epoch 29][40/200] loss=0.615101 avg=0.665749 VRAM=38.5GiB | 5.8% done | ETA(epoch): 2123s + [Epoch 29][50/200] loss=0.573402 avg=0.655386 VRAM=38.6GiB | 5.9% done | ETA(epoch): 1991s + [Epoch 29][60/200] loss=0.589283 avg=0.653027 VRAM=38.5GiB | 5.9% done | ETA(epoch): 1858s + [Epoch 29][70/200] loss=0.634191 avg=0.651159 VRAM=38.6GiB | 5.9% done | ETA(epoch): 1725s + [Epoch 29][80/200] loss=0.626339 avg=0.656503 VRAM=38.5GiB | 5.9% done | ETA(epoch): 1592s + [Epoch 29][90/200] loss=0.614846 avg=0.658560 VRAM=38.6GiB | 5.9% done | ETA(epoch): 1463s + [Epoch 29][100/200] loss=0.612545 avg=0.658984 VRAM=38.5GiB | 5.9% done | ETA(epoch): 1330s + [Epoch 29][110/200] loss=0.693801 avg=0.661540 VRAM=38.6GiB | 5.9% done | ETA(epoch): 1196s + [Epoch 29][120/200] loss=0.642560 avg=0.659292 VRAM=38.5GiB | 5.9% done | ETA(epoch): 1063s + [Epoch 29][130/200] loss=0.716193 avg=0.660127 VRAM=38.6GiB | 5.9% done | ETA(epoch): 930s + [Epoch 29][140/200] loss=0.652043 avg=0.657705 VRAM=38.5GiB | 5.9% done | ETA(epoch): 797s + [Epoch 29][150/200] loss=0.677698 avg=0.660171 VRAM=38.6GiB | 5.9% done | ETA(epoch): 664s + [Epoch 29][160/200] loss=0.676543 avg=0.660033 VRAM=38.5GiB | 6.0% done | ETA(epoch): 531s + [Epoch 29][170/200] loss=0.596112 avg=0.659352 VRAM=38.6GiB | 6.0% done | ETA(epoch): 399s + [Epoch 29][180/200] loss=0.628986 avg=0.659432 VRAM=38.5GiB | 6.0% done | ETA(epoch): 266s + [Epoch 29][190/200] loss=0.650739 avg=0.659389 VRAM=38.6GiB | 6.0% done | ETA(epoch): 133s + [Epoch 29][200/200] loss=0.789035 avg=0.659365 VRAM=38.5GiB | 6.0% done | ETA(epoch): 0s + Train loss: 0.659365 (2655.9s) ETA: 23250min + Val loss: 0.713805 [t_0.0-0.2=0.9381 t_0.2-0.4=0.7711 t_0.4-0.6=0.6608 t_0.6-0.8=0.5762 t_0.8-1.0=0.5906] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 30/499 (6% done) --- + [Epoch 30][10/200] loss=0.699308 avg=0.660642 VRAM=38.6GiB | 6.0% done | ETA(epoch): 2516s + [Epoch 30][20/200] loss=0.713849 avg=0.663739 VRAM=38.5GiB | 6.0% done | ETA(epoch): 2387s + [Epoch 30][30/200] loss=0.743017 avg=0.657894 VRAM=38.6GiB | 6.0% done | ETA(epoch): 2255s + [Epoch 30][40/200] loss=0.624946 avg=0.659848 VRAM=38.5GiB | 6.0% done | ETA(epoch): 2123s + [Epoch 30][50/200] loss=0.559906 avg=0.654192 VRAM=38.6GiB | 6.0% done | ETA(epoch): 1990s + [Epoch 30][60/200] loss=0.658380 avg=0.655114 VRAM=38.5GiB | 6.1% done | ETA(epoch): 1858s + [Epoch 30][70/200] loss=0.639429 avg=0.656876 VRAM=38.6GiB | 6.1% done | ETA(epoch): 1725s + [Epoch 30][80/200] loss=0.569007 avg=0.654397 VRAM=38.5GiB | 6.1% done | ETA(epoch): 1593s + [Epoch 30][90/200] loss=0.685014 avg=0.653605 VRAM=38.6GiB | 6.1% done | ETA(epoch): 1460s + [Epoch 30][100/200] loss=0.598334 avg=0.654063 VRAM=38.5GiB | 6.1% done | ETA(epoch): 1327s + [Epoch 30][110/200] loss=0.653746 avg=0.655483 VRAM=38.6GiB | 6.1% done | ETA(epoch): 1194s + [Epoch 30][120/200] loss=0.736132 avg=0.654766 VRAM=38.5GiB | 6.1% done | ETA(epoch): 1061s + [Epoch 30][130/200] loss=0.594488 avg=0.657345 VRAM=38.6GiB | 6.1% done | ETA(epoch): 929s + [Epoch 30][140/200] loss=0.609977 avg=0.658238 VRAM=38.5GiB | 6.1% done | ETA(epoch): 796s + [Epoch 30][150/200] loss=0.599441 avg=0.657382 VRAM=38.6GiB | 6.2% done | ETA(epoch): 664s + [Epoch 30][160/200] loss=0.689900 avg=0.657374 VRAM=38.5GiB | 6.2% done | ETA(epoch): 531s + [Epoch 30][170/200] loss=0.640730 avg=0.656643 VRAM=38.6GiB | 6.2% done | ETA(epoch): 398s + [Epoch 30][180/200] loss=0.715292 avg=0.658444 VRAM=38.5GiB | 6.2% done | ETA(epoch): 265s + [Epoch 30][190/200] loss=0.561413 avg=0.658961 VRAM=38.6GiB | 6.2% done | ETA(epoch): 133s + [Epoch 30][200/200] loss=0.736875 avg=0.658651 VRAM=38.5GiB | 6.2% done | ETA(epoch): 0s + Train loss: 0.658651 (2654.4s) ETA: 23184min + Val loss: 0.692570 [t_0.0-0.2=0.9410 t_0.2-0.4=0.7506 t_0.4-0.6=0.6329 t_0.6-0.8=0.5753 t_0.8-1.0=0.5920] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 31/499 (6% done) --- + [Epoch 31][10/200] loss=0.721543 avg=0.673929 VRAM=38.6GiB | 6.2% done | ETA(epoch): 2523s + [Epoch 31][20/200] loss=0.641764 avg=0.656104 VRAM=38.5GiB | 6.2% done | ETA(epoch): 2391s + [Epoch 31][30/200] loss=0.626202 avg=0.649691 VRAM=38.6GiB | 6.2% done | ETA(epoch): 2258s + [Epoch 31][40/200] loss=0.707940 avg=0.653423 VRAM=38.5GiB | 6.2% done | ETA(epoch): 2125s + [Epoch 31][50/200] loss=0.636415 avg=0.658143 VRAM=38.6GiB | 6.2% done | ETA(epoch): 1991s + [Epoch 31][60/200] loss=0.644560 avg=0.661962 VRAM=38.5GiB | 6.3% done | ETA(epoch): 1858s + [Epoch 31][70/200] loss=0.720072 avg=0.664207 VRAM=38.6GiB | 6.3% done | ETA(epoch): 1726s + [Epoch 31][80/200] loss=0.774008 avg=0.662352 VRAM=38.5GiB | 6.3% done | ETA(epoch): 1593s + [Epoch 31][90/200] loss=0.601932 avg=0.667900 VRAM=38.6GiB | 6.3% done | ETA(epoch): 1460s + [Epoch 31][100/200] loss=0.734885 avg=0.671394 VRAM=38.5GiB | 6.3% done | ETA(epoch): 1327s + [Epoch 31][110/200] loss=0.589936 avg=0.673346 VRAM=38.6GiB | 6.3% done | ETA(epoch): 1194s + [Epoch 31][120/200] loss=0.626119 avg=0.672671 VRAM=38.5GiB | 6.3% done | ETA(epoch): 1062s + [Epoch 31][130/200] loss=0.706598 avg=0.672529 VRAM=38.6GiB | 6.3% done | ETA(epoch): 929s + [Epoch 31][140/200] loss=0.646645 avg=0.671811 VRAM=38.5GiB | 6.3% done | ETA(epoch): 796s + [Epoch 31][150/200] loss=0.661555 avg=0.671613 VRAM=38.6GiB | 6.3% done | ETA(epoch): 663s + [Epoch 31][160/200] loss=0.669915 avg=0.670448 VRAM=38.5GiB | 6.4% done | ETA(epoch): 531s + [Epoch 31][170/200] loss=0.730697 avg=0.670977 VRAM=38.6GiB | 6.4% done | ETA(epoch): 398s + [Epoch 31][180/200] loss=0.645820 avg=0.670782 VRAM=38.5GiB | 6.4% done | ETA(epoch): 265s + [Epoch 31][190/200] loss=0.711676 avg=0.671101 VRAM=38.6GiB | 6.4% done | ETA(epoch): 133s + [Epoch 31][200/200] loss=0.746935 avg=0.671045 VRAM=38.5GiB | 6.4% done | ETA(epoch): 0s + Train loss: 0.671045 (2653.4s) ETA: 23119min + Val loss: 0.702577 [t_0.0-0.2=0.9250 t_0.2-0.4=0.7530 t_0.4-0.6=0.6605 t_0.6-0.8=0.5519 t_0.8-1.0=0.5914] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 32/499 (6% done) --- + [Epoch 32][10/200] loss=0.723397 avg=0.673973 VRAM=38.6GiB | 6.4% done | ETA(epoch): 2517s + [Epoch 32][20/200] loss=0.631053 avg=0.658127 VRAM=38.5GiB | 6.4% done | ETA(epoch): 2387s + [Epoch 32][30/200] loss=0.640515 avg=0.660604 VRAM=38.6GiB | 6.4% done | ETA(epoch): 2255s + [Epoch 32][40/200] loss=0.703102 avg=0.650728 VRAM=38.5GiB | 6.4% done | ETA(epoch): 2122s + [Epoch 32][50/200] loss=0.587990 avg=0.659092 VRAM=38.6GiB | 6.5% done | ETA(epoch): 1990s + [Epoch 32][60/200] loss=0.592651 avg=0.661953 VRAM=38.5GiB | 6.5% done | ETA(epoch): 1857s + [Epoch 32][70/200] loss=0.664480 avg=0.662174 VRAM=38.6GiB | 6.5% done | ETA(epoch): 1729s + [Epoch 32][80/200] loss=0.578453 avg=0.657777 VRAM=38.5GiB | 6.5% done | ETA(epoch): 1596s + [Epoch 32][90/200] loss=0.592923 avg=0.657341 VRAM=38.6GiB | 6.5% done | ETA(epoch): 1462s + [Epoch 32][100/200] loss=0.602398 avg=0.658769 VRAM=38.5GiB | 6.5% done | ETA(epoch): 1330s + [Epoch 32][110/200] loss=0.711352 avg=0.657718 VRAM=38.6GiB | 6.5% done | ETA(epoch): 1196s + [Epoch 32][120/200] loss=0.567503 avg=0.656379 VRAM=38.5GiB | 6.5% done | ETA(epoch): 1063s + [Epoch 32][130/200] loss=0.703676 avg=0.660214 VRAM=38.6GiB | 6.5% done | ETA(epoch): 930s + [Epoch 32][140/200] loss=0.732835 avg=0.662493 VRAM=38.5GiB | 6.5% done | ETA(epoch): 797s + [Epoch 32][150/200] loss=0.629623 avg=0.662280 VRAM=38.6GiB | 6.6% done | ETA(epoch): 664s + [Epoch 32][160/200] loss=0.732802 avg=0.659848 VRAM=38.5GiB | 6.6% done | ETA(epoch): 531s + [Epoch 32][170/200] loss=0.628231 avg=0.659099 VRAM=38.6GiB | 6.6% done | ETA(epoch): 399s + [Epoch 32][180/200] loss=0.808646 avg=0.659737 VRAM=38.5GiB | 6.6% done | ETA(epoch): 266s + [Epoch 32][190/200] loss=0.588228 avg=0.660071 VRAM=38.6GiB | 6.6% done | ETA(epoch): 133s + [Epoch 32][200/200] loss=0.554973 avg=0.658222 VRAM=38.5GiB | 6.6% done | ETA(epoch): 0s + Train loss: 0.658222 (2656.3s) ETA: 23056min + Val loss: 0.699655 [t_0.0-0.2=0.9202 t_0.2-0.4=0.7649 t_0.4-0.6=0.6625 t_0.6-0.8=0.5741 t_0.8-1.0=0.5865] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 33/499 (7% done) --- + [Epoch 33][10/200] loss=0.723706 avg=0.657065 VRAM=38.6GiB | 6.6% done | ETA(epoch): 2518s + [Epoch 33][20/200] loss=0.641497 avg=0.655689 VRAM=38.5GiB | 6.6% done | ETA(epoch): 2387s + [Epoch 33][30/200] loss=0.696604 avg=0.656496 VRAM=38.6GiB | 6.6% done | ETA(epoch): 2254s + [Epoch 33][40/200] loss=0.602698 avg=0.659105 VRAM=38.5GiB | 6.6% done | ETA(epoch): 2122s + [Epoch 33][50/200] loss=0.682563 avg=0.656466 VRAM=38.6GiB | 6.7% done | ETA(epoch): 1989s + [Epoch 33][60/200] loss=0.595850 avg=0.658759 VRAM=38.5GiB | 6.7% done | ETA(epoch): 1857s + [Epoch 33][70/200] loss=0.655611 avg=0.655902 VRAM=38.6GiB | 6.7% done | ETA(epoch): 1724s + [Epoch 33][80/200] loss=0.603242 avg=0.655488 VRAM=38.5GiB | 6.7% done | ETA(epoch): 1592s + [Epoch 33][90/200] loss=0.607014 avg=0.658580 VRAM=38.6GiB | 6.7% done | ETA(epoch): 1459s + [Epoch 33][100/200] loss=0.673967 avg=0.657101 VRAM=38.5GiB | 6.7% done | ETA(epoch): 1326s + [Epoch 33][110/200] loss=0.673170 avg=0.660710 VRAM=38.6GiB | 6.7% done | ETA(epoch): 1193s + [Epoch 33][120/200] loss=0.629591 avg=0.664494 VRAM=38.5GiB | 6.7% done | ETA(epoch): 1061s + [Epoch 33][130/200] loss=0.661737 avg=0.664754 VRAM=38.6GiB | 6.7% done | ETA(epoch): 928s + [Epoch 33][140/200] loss=0.648283 avg=0.663864 VRAM=38.5GiB | 6.7% done | ETA(epoch): 796s + [Epoch 33][150/200] loss=0.633090 avg=0.668558 VRAM=38.6GiB | 6.8% done | ETA(epoch): 663s + [Epoch 33][160/200] loss=0.718046 avg=0.667855 VRAM=38.5GiB | 6.8% done | ETA(epoch): 530s + [Epoch 33][170/200] loss=0.680257 avg=0.667966 VRAM=38.6GiB | 6.8% done | ETA(epoch): 398s + [Epoch 33][180/200] loss=0.752970 avg=0.668670 VRAM=38.5GiB | 6.8% done | ETA(epoch): 265s + [Epoch 33][190/200] loss=0.713722 avg=0.668295 VRAM=38.6GiB | 6.8% done | ETA(epoch): 133s + [Epoch 33][200/200] loss=0.615571 avg=0.667728 VRAM=38.5GiB | 6.8% done | ETA(epoch): 0s + Train loss: 0.667728 (2651.8s) ETA: 22992min + Val loss: 0.708469 [t_0.0-0.2=0.9487 t_0.2-0.4=0.7601 t_0.4-0.6=0.6416 t_0.6-0.8=0.5764 t_0.8-1.0=0.5901] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 34/499 (7% done) --- + [Epoch 34][10/200] loss=0.684333 avg=0.680742 VRAM=38.6GiB | 6.8% done | ETA(epoch): 2518s + [Epoch 34][20/200] loss=0.572269 avg=0.660846 VRAM=38.5GiB | 6.8% done | ETA(epoch): 2386s + [Epoch 34][30/200] loss=0.684603 avg=0.664079 VRAM=38.6GiB | 6.8% done | ETA(epoch): 2255s + [Epoch 34][40/200] loss=0.602079 avg=0.651314 VRAM=38.5GiB | 6.8% done | ETA(epoch): 2122s + [Epoch 34][50/200] loss=0.583209 avg=0.649160 VRAM=38.6GiB | 6.9% done | ETA(epoch): 1989s + [Epoch 34][60/200] loss=0.663682 avg=0.656170 VRAM=38.5GiB | 6.9% done | ETA(epoch): 1856s + [Epoch 34][70/200] loss=0.568279 avg=0.656324 VRAM=38.6GiB | 6.9% done | ETA(epoch): 1724s + [Epoch 34][80/200] loss=0.804188 avg=0.660398 VRAM=38.5GiB | 6.9% done | ETA(epoch): 1591s + [Epoch 34][90/200] loss=0.741882 avg=0.662582 VRAM=38.6GiB | 6.9% done | ETA(epoch): 1459s + [Epoch 34][100/200] loss=0.590009 avg=0.663655 VRAM=38.5GiB | 6.9% done | ETA(epoch): 1326s + [Epoch 34][110/200] loss=0.756638 avg=0.663102 VRAM=38.6GiB | 6.9% done | ETA(epoch): 1193s + [Epoch 34][120/200] loss=0.787434 avg=0.666704 VRAM=38.5GiB | 6.9% done | ETA(epoch): 1061s + [Epoch 34][130/200] loss=0.669110 avg=0.667020 VRAM=38.6GiB | 6.9% done | ETA(epoch): 928s + [Epoch 34][140/200] loss=0.705750 avg=0.666997 VRAM=38.5GiB | 6.9% done | ETA(epoch): 796s + [Epoch 34][150/200] loss=0.624787 avg=0.669110 VRAM=38.6GiB | 7.0% done | ETA(epoch): 663s + [Epoch 34][160/200] loss=0.677729 avg=0.668959 VRAM=38.5GiB | 7.0% done | ETA(epoch): 530s + [Epoch 34][170/200] loss=0.721381 avg=0.669598 VRAM=38.6GiB | 7.0% done | ETA(epoch): 398s + [Epoch 34][180/200] loss=0.628578 avg=0.669030 VRAM=38.5GiB | 7.0% done | ETA(epoch): 265s + [Epoch 34][190/200] loss=0.615233 avg=0.668692 VRAM=38.6GiB | 7.0% done | ETA(epoch): 133s + [Epoch 34][200/200] loss=0.606226 avg=0.667097 VRAM=38.5GiB | 7.0% done | ETA(epoch): 0s + Train loss: 0.667097 (2652.4s) ETA: 22931min + Val loss: 0.712310 [t_0.0-0.2=0.9320 t_0.2-0.4=0.7607 t_0.4-0.6=0.6547 t_0.6-0.8=0.5555 t_0.8-1.0=0.5857] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 35/499 (7% done) --- + [Epoch 35][10/200] loss=0.668632 avg=0.699457 VRAM=38.6GiB | 7.0% done | ETA(epoch): 2521s + [Epoch 35][20/200] loss=0.652662 avg=0.691254 VRAM=38.5GiB | 7.0% done | ETA(epoch): 2387s + [Epoch 35][30/200] loss=0.594721 avg=0.676063 VRAM=38.6GiB | 7.0% done | ETA(epoch): 2256s + [Epoch 35][40/200] loss=0.730333 avg=0.674592 VRAM=38.5GiB | 7.0% done | ETA(epoch): 2122s + [Epoch 35][50/200] loss=0.711276 avg=0.667476 VRAM=38.6GiB | 7.0% done | ETA(epoch): 2000s + [Epoch 35][60/200] loss=0.623562 avg=0.665417 VRAM=38.5GiB | 7.1% done | ETA(epoch): 1864s + [Epoch 35][70/200] loss=0.615048 avg=0.666555 VRAM=38.6GiB | 7.1% done | ETA(epoch): 1730s + [Epoch 35][80/200] loss=0.670783 avg=0.666160 VRAM=38.5GiB | 7.1% done | ETA(epoch): 1596s + [Epoch 35][90/200] loss=0.713643 avg=0.668724 VRAM=38.6GiB | 7.1% done | ETA(epoch): 1463s + [Epoch 35][100/200] loss=0.548041 avg=0.672819 VRAM=38.5GiB | 7.1% done | ETA(epoch): 1329s + [Epoch 35][110/200] loss=0.774068 avg=0.673929 VRAM=38.6GiB | 7.1% done | ETA(epoch): 1196s + [Epoch 35][120/200] loss=0.777067 avg=0.676328 VRAM=38.5GiB | 7.1% done | ETA(epoch): 1063s + [Epoch 35][130/200] loss=0.653276 avg=0.673790 VRAM=38.6GiB | 7.1% done | ETA(epoch): 930s + [Epoch 35][140/200] loss=0.729657 avg=0.674512 VRAM=38.5GiB | 7.1% done | ETA(epoch): 797s + [Epoch 35][150/200] loss=0.692164 avg=0.674747 VRAM=38.6GiB | 7.1% done | ETA(epoch): 664s + [Epoch 35][160/200] loss=0.760109 avg=0.672659 VRAM=38.5GiB | 7.2% done | ETA(epoch): 531s + [Epoch 35][170/200] loss=0.714980 avg=0.671392 VRAM=38.6GiB | 7.2% done | ETA(epoch): 398s + [Epoch 35][180/200] loss=0.687966 avg=0.670204 VRAM=38.5GiB | 7.2% done | ETA(epoch): 266s + [Epoch 35][190/200] loss=0.702556 avg=0.669901 VRAM=38.6GiB | 7.2% done | ETA(epoch): 133s + [Epoch 35][200/200] loss=0.703737 avg=0.669477 VRAM=38.5GiB | 7.2% done | ETA(epoch): 0s + Train loss: 0.669477 (2655.8s) ETA: 22870min + Val loss: 0.698215 [t_0.0-0.2=0.9532 t_0.2-0.4=0.7470 t_0.4-0.6=0.6383 t_0.6-0.8=0.5531 t_0.8-1.0=0.5961] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 36/499 (7% done) --- + [Epoch 36][10/200] loss=0.678386 avg=0.663031 VRAM=38.6GiB | 7.2% done | ETA(epoch): 2514s + [Epoch 36][20/200] loss=0.636231 avg=0.658185 VRAM=38.5GiB | 7.2% done | ETA(epoch): 2385s + [Epoch 36][30/200] loss=0.750602 avg=0.659542 VRAM=38.6GiB | 7.2% done | ETA(epoch): 2253s + [Epoch 36][40/200] loss=0.602387 avg=0.660538 VRAM=38.5GiB | 7.2% done | ETA(epoch): 2121s + [Epoch 36][50/200] loss=0.701523 avg=0.658838 VRAM=38.6GiB | 7.2% done | ETA(epoch): 1988s + [Epoch 36][60/200] loss=0.653232 avg=0.660194 VRAM=38.5GiB | 7.3% done | ETA(epoch): 1856s + [Epoch 36][70/200] loss=0.678309 avg=0.662808 VRAM=38.6GiB | 7.3% done | ETA(epoch): 1724s + [Epoch 36][80/200] loss=0.568375 avg=0.664667 VRAM=38.5GiB | 7.3% done | ETA(epoch): 1591s + [Epoch 36][90/200] loss=0.821248 avg=0.668906 VRAM=38.6GiB | 7.3% done | ETA(epoch): 1459s + [Epoch 36][100/200] loss=0.527310 avg=0.664974 VRAM=38.5GiB | 7.3% done | ETA(epoch): 1326s + [Epoch 36][110/200] loss=0.610570 avg=0.664506 VRAM=38.6GiB | 7.3% done | ETA(epoch): 1194s + [Epoch 36][120/200] loss=0.739877 avg=0.662371 VRAM=38.5GiB | 7.3% done | ETA(epoch): 1061s + [Epoch 36][130/200] loss=0.596045 avg=0.662212 VRAM=38.6GiB | 7.3% done | ETA(epoch): 928s + [Epoch 36][140/200] loss=0.692797 avg=0.662882 VRAM=38.5GiB | 7.3% done | ETA(epoch): 796s + [Epoch 36][150/200] loss=0.656048 avg=0.663242 VRAM=38.6GiB | 7.3% done | ETA(epoch): 663s + [Epoch 36][160/200] loss=0.667676 avg=0.662966 VRAM=38.5GiB | 7.4% done | ETA(epoch): 530s + [Epoch 36][170/200] loss=0.609905 avg=0.664230 VRAM=38.6GiB | 7.4% done | ETA(epoch): 398s + [Epoch 36][180/200] loss=0.700820 avg=0.664210 VRAM=38.5GiB | 7.4% done | ETA(epoch): 265s + [Epoch 36][190/200] loss=0.658092 avg=0.664452 VRAM=38.6GiB | 7.4% done | ETA(epoch): 133s + [Epoch 36][200/200] loss=0.628656 avg=0.663104 VRAM=38.5GiB | 7.4% done | ETA(epoch): 0s + Train loss: 0.663104 (2654.7s) ETA: 22809min + Val loss: 0.698700 [t_0.0-0.2=0.9448 t_0.2-0.4=0.7439 t_0.4-0.6=0.6510 t_0.6-0.8=0.5707 t_0.8-1.0=0.5866] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 37/499 (7% done) --- + [Epoch 37][10/200] loss=0.636576 avg=0.672208 VRAM=38.6GiB | 7.4% done | ETA(epoch): 2522s + [Epoch 37][20/200] loss=0.720862 avg=0.671298 VRAM=38.5GiB | 7.4% done | ETA(epoch): 2387s + [Epoch 37][30/200] loss=0.585967 avg=0.656653 VRAM=38.6GiB | 7.4% done | ETA(epoch): 2256s + [Epoch 37][40/200] loss=0.690551 avg=0.660926 VRAM=38.5GiB | 7.4% done | ETA(epoch): 2123s + [Epoch 37][50/200] loss=0.657183 avg=0.656571 VRAM=38.6GiB | 7.4% done | ETA(epoch): 1990s + [Epoch 37][60/200] loss=0.680544 avg=0.657918 VRAM=38.5GiB | 7.5% done | ETA(epoch): 1857s + [Epoch 37][70/200] loss=0.719104 avg=0.663584 VRAM=38.6GiB | 7.5% done | ETA(epoch): 1724s + [Epoch 37][80/200] loss=0.670915 avg=0.661458 VRAM=38.5GiB | 7.5% done | ETA(epoch): 1592s + [Epoch 37][90/200] loss=0.564993 avg=0.660547 VRAM=38.6GiB | 7.5% done | ETA(epoch): 1459s + [Epoch 37][100/200] loss=0.604877 avg=0.665340 VRAM=38.5GiB | 7.5% done | ETA(epoch): 1327s + [Epoch 37][110/200] loss=0.629669 avg=0.664063 VRAM=38.6GiB | 7.5% done | ETA(epoch): 1194s + [Epoch 37][120/200] loss=0.727284 avg=0.664466 VRAM=38.5GiB | 7.5% done | ETA(epoch): 1061s + [Epoch 37][130/200] loss=0.793548 avg=0.662603 VRAM=38.6GiB | 7.5% done | ETA(epoch): 929s + [Epoch 37][140/200] loss=0.755038 avg=0.666563 VRAM=38.5GiB | 7.5% done | ETA(epoch): 796s + [Epoch 37][150/200] loss=0.601543 avg=0.666415 VRAM=38.6GiB | 7.5% done | ETA(epoch): 663s + [Epoch 37][160/200] loss=0.706048 avg=0.667630 VRAM=38.5GiB | 7.6% done | ETA(epoch): 531s + [Epoch 37][170/200] loss=0.652919 avg=0.668551 VRAM=38.6GiB | 7.6% done | ETA(epoch): 398s + [Epoch 37][180/200] loss=0.693286 avg=0.667729 VRAM=38.5GiB | 7.6% done | ETA(epoch): 265s + [Epoch 37][190/200] loss=0.699426 avg=0.666809 VRAM=38.6GiB | 7.6% done | ETA(epoch): 133s + [Epoch 37][200/200] loss=0.665793 avg=0.666530 VRAM=38.5GiB | 7.6% done | ETA(epoch): 0s + Train loss: 0.666530 (2652.9s) ETA: 22749min + Val loss: 0.691540 [t_0.0-0.2=0.9292 t_0.2-0.4=0.7585 t_0.4-0.6=0.6408 t_0.6-0.8=0.5705 t_0.8-1.0=0.5757] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 38/499 (8% done) --- + [Epoch 38][10/200] loss=0.766068 avg=0.690087 VRAM=38.6GiB | 7.6% done | ETA(epoch): 2517s + [Epoch 38][20/200] loss=0.609908 avg=0.657750 VRAM=38.5GiB | 7.6% done | ETA(epoch): 2385s + [Epoch 38][30/200] loss=0.645830 avg=0.667564 VRAM=38.6GiB | 7.6% done | ETA(epoch): 2254s + [Epoch 38][40/200] loss=0.646225 avg=0.674942 VRAM=38.5GiB | 7.6% done | ETA(epoch): 2121s + [Epoch 38][50/200] loss=0.677511 avg=0.671263 VRAM=38.6GiB | 7.6% done | ETA(epoch): 1989s + [Epoch 38][60/200] loss=0.770320 avg=0.669318 VRAM=38.5GiB | 7.7% done | ETA(epoch): 1856s + [Epoch 38][70/200] loss=0.786206 avg=0.664500 VRAM=38.6GiB | 7.7% done | ETA(epoch): 1724s + [Epoch 38][80/200] loss=0.618816 avg=0.667942 VRAM=38.5GiB | 7.7% done | ETA(epoch): 1591s + [Epoch 38][90/200] loss=0.674005 avg=0.667709 VRAM=38.6GiB | 7.7% done | ETA(epoch): 1459s + [Epoch 38][100/200] loss=0.760163 avg=0.665601 VRAM=38.5GiB | 7.7% done | ETA(epoch): 1326s + [Epoch 38][110/200] loss=0.916467 avg=0.670503 VRAM=38.6GiB | 7.7% done | ETA(epoch): 1193s + [Epoch 38][120/200] loss=0.617860 avg=0.671815 VRAM=38.5GiB | 7.7% done | ETA(epoch): 1063s + [Epoch 38][130/200] loss=0.619855 avg=0.669463 VRAM=38.6GiB | 7.7% done | ETA(epoch): 930s + [Epoch 38][140/200] loss=0.651547 avg=0.668177 VRAM=38.5GiB | 7.7% done | ETA(epoch): 797s + [Epoch 38][150/200] loss=0.580574 avg=0.669126 VRAM=38.6GiB | 7.8% done | ETA(epoch): 664s + [Epoch 38][160/200] loss=0.740144 avg=0.668448 VRAM=38.5GiB | 7.8% done | ETA(epoch): 531s + [Epoch 38][170/200] loss=0.671076 avg=0.667276 VRAM=38.6GiB | 7.8% done | ETA(epoch): 398s + [Epoch 38][180/200] loss=0.671545 avg=0.666873 VRAM=38.5GiB | 7.8% done | ETA(epoch): 266s + [Epoch 38][190/200] loss=0.719346 avg=0.665439 VRAM=38.6GiB | 7.8% done | ETA(epoch): 133s + [Epoch 38][200/200] loss=0.704955 avg=0.665740 VRAM=38.5GiB | 7.8% done | ETA(epoch): 0s + Train loss: 0.665740 (2654.8s) ETA: 22690min + Val loss: 0.708362 [t_0.0-0.2=0.9282 t_0.2-0.4=0.7649 t_0.4-0.6=0.6564 t_0.6-0.8=0.5500 t_0.8-1.0=0.5977] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 39/499 (8% done) --- + [Epoch 39][10/200] loss=0.587844 avg=0.648057 VRAM=38.6GiB | 7.8% done | ETA(epoch): 2517s + [Epoch 39][20/200] loss=0.645744 avg=0.670901 VRAM=38.5GiB | 7.8% done | ETA(epoch): 2386s + [Epoch 39][30/200] loss=0.656591 avg=0.670611 VRAM=38.6GiB | 7.8% done | ETA(epoch): 2253s + [Epoch 39][40/200] loss=0.631041 avg=0.669356 VRAM=38.5GiB | 7.8% done | ETA(epoch): 2121s + [Epoch 39][50/200] loss=0.631146 avg=0.664653 VRAM=38.6GiB | 7.8% done | ETA(epoch): 1988s + [Epoch 39][60/200] loss=0.600027 avg=0.669298 VRAM=38.5GiB | 7.9% done | ETA(epoch): 1856s + [Epoch 39][70/200] loss=0.580291 avg=0.662422 VRAM=38.6GiB | 7.9% done | ETA(epoch): 1724s + [Epoch 39][80/200] loss=0.753068 avg=0.668158 VRAM=38.5GiB | 7.9% done | ETA(epoch): 1595s + [Epoch 39][90/200] loss=0.602582 avg=0.665724 VRAM=38.6GiB | 7.9% done | ETA(epoch): 1462s + [Epoch 39][100/200] loss=0.773557 avg=0.667948 VRAM=38.5GiB | 7.9% done | ETA(epoch): 1329s + [Epoch 39][110/200] loss=0.603707 avg=0.666144 VRAM=38.6GiB | 7.9% done | ETA(epoch): 1196s + [Epoch 39][120/200] loss=0.602901 avg=0.663519 VRAM=38.5GiB | 7.9% done | ETA(epoch): 1063s + [Epoch 39][130/200] loss=0.556341 avg=0.661746 VRAM=38.6GiB | 7.9% done | ETA(epoch): 930s + [Epoch 39][140/200] loss=0.688947 avg=0.659540 VRAM=38.5GiB | 7.9% done | ETA(epoch): 797s + [Epoch 39][150/200] loss=0.597806 avg=0.657162 VRAM=38.6GiB | 8.0% done | ETA(epoch): 664s + [Epoch 39][160/200] loss=0.628802 avg=0.656259 VRAM=38.5GiB | 8.0% done | ETA(epoch): 531s + [Epoch 39][170/200] loss=0.611219 avg=0.658377 VRAM=38.6GiB | 8.0% done | ETA(epoch): 398s + [Epoch 39][180/200] loss=0.701784 avg=0.659536 VRAM=38.5GiB | 8.0% done | ETA(epoch): 266s + [Epoch 39][190/200] loss=0.633293 avg=0.657847 VRAM=38.6GiB | 8.0% done | ETA(epoch): 133s + [Epoch 39][200/200] loss=0.754534 avg=0.657723 VRAM=38.5GiB | 8.0% done | ETA(epoch): 0s + Train loss: 0.657723 (2656.0s) ETA: 22632min + Val loss: 0.700144 [t_0.0-0.2=0.9167 t_0.2-0.4=0.7601 t_0.4-0.6=0.6538 t_0.6-0.8=0.5741 t_0.8-1.0=0.5808] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 40/499 (8% done) --- + [Epoch 40][10/200] loss=0.595212 avg=0.673095 VRAM=38.6GiB | 8.0% done | ETA(epoch): 2515s + [Epoch 40][20/200] loss=0.666523 avg=0.671263 VRAM=38.5GiB | 8.0% done | ETA(epoch): 2386s + [Epoch 40][30/200] loss=0.574940 avg=0.669327 VRAM=38.6GiB | 8.0% done | ETA(epoch): 2252s + [Epoch 40][40/200] loss=0.783044 avg=0.670424 VRAM=38.5GiB | 8.0% done | ETA(epoch): 2121s + [Epoch 40][50/200] loss=0.798353 avg=0.671700 VRAM=38.6GiB | 8.1% done | ETA(epoch): 1991s + [Epoch 40][60/200] loss=0.644542 avg=0.668441 VRAM=38.5GiB | 8.1% done | ETA(epoch): 1857s + [Epoch 40][70/200] loss=0.581735 avg=0.665840 VRAM=38.6GiB | 8.1% done | ETA(epoch): 1725s + [Epoch 40][80/200] loss=0.591392 avg=0.667167 VRAM=38.5GiB | 8.1% done | ETA(epoch): 1592s + [Epoch 40][90/200] loss=0.705801 avg=0.669811 VRAM=38.6GiB | 8.1% done | ETA(epoch): 1459s + [Epoch 40][100/200] loss=0.577337 avg=0.668277 VRAM=38.5GiB | 8.1% done | ETA(epoch): 1327s + [Epoch 40][110/200] loss=0.753046 avg=0.667785 VRAM=38.6GiB | 8.1% done | ETA(epoch): 1194s + [Epoch 40][120/200] loss=0.699436 avg=0.666928 VRAM=38.5GiB | 8.1% done | ETA(epoch): 1061s + [Epoch 40][130/200] loss=0.680070 avg=0.665244 VRAM=38.6GiB | 8.1% done | ETA(epoch): 928s + [Epoch 40][140/200] loss=0.548687 avg=0.666354 VRAM=38.5GiB | 8.1% done | ETA(epoch): 796s + [Epoch 40][150/200] loss=0.541892 avg=0.665421 VRAM=38.6GiB | 8.2% done | ETA(epoch): 663s + [Epoch 40][160/200] loss=0.611797 avg=0.663385 VRAM=38.5GiB | 8.2% done | ETA(epoch): 531s + [Epoch 40][170/200] loss=0.644680 avg=0.661818 VRAM=38.6GiB | 8.2% done | ETA(epoch): 398s + [Epoch 40][180/200] loss=0.577765 avg=0.662046 VRAM=38.5GiB | 8.2% done | ETA(epoch): 265s + [Epoch 40][190/200] loss=0.681060 avg=0.663259 VRAM=38.6GiB | 8.2% done | ETA(epoch): 133s + [Epoch 40][200/200] loss=0.655969 avg=0.663375 VRAM=38.5GiB | 8.2% done | ETA(epoch): 0s + Train loss: 0.663375 (2652.7s) ETA: 22574min + Val loss: 0.715077 [t_0.0-0.2=0.9381 t_0.2-0.4=0.7532 t_0.4-0.6=0.6503 t_0.6-0.8=0.5666 t_0.8-1.0=0.5793] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 41/499 (8% done) --- + [Epoch 41][10/200] loss=0.793043 avg=0.672699 VRAM=38.6GiB | 8.2% done | ETA(epoch): 2517s + [Epoch 41][20/200] loss=0.767394 avg=0.651426 VRAM=38.5GiB | 8.2% done | ETA(epoch): 2389s + [Epoch 41][30/200] loss=0.716474 avg=0.655709 VRAM=38.6GiB | 8.2% done | ETA(epoch): 2255s + [Epoch 41][40/200] loss=0.590460 avg=0.655022 VRAM=38.5GiB | 8.2% done | ETA(epoch): 2122s + [Epoch 41][50/200] loss=0.618161 avg=0.654929 VRAM=38.6GiB | 8.2% done | ETA(epoch): 1990s + [Epoch 41][60/200] loss=0.702249 avg=0.654031 VRAM=38.5GiB | 8.3% done | ETA(epoch): 1857s + [Epoch 41][70/200] loss=0.720695 avg=0.657178 VRAM=38.6GiB | 8.3% done | ETA(epoch): 1724s + [Epoch 41][80/200] loss=0.686803 avg=0.658859 VRAM=38.5GiB | 8.3% done | ETA(epoch): 1592s + [Epoch 41][90/200] loss=0.685519 avg=0.659539 VRAM=38.6GiB | 8.3% done | ETA(epoch): 1459s + [Epoch 41][100/200] loss=0.746911 avg=0.659454 VRAM=38.5GiB | 8.3% done | ETA(epoch): 1326s + [Epoch 41][110/200] loss=0.678086 avg=0.656316 VRAM=38.6GiB | 8.3% done | ETA(epoch): 1194s + [Epoch 41][120/200] loss=0.663195 avg=0.656125 VRAM=38.5GiB | 8.3% done | ETA(epoch): 1061s + [Epoch 41][130/200] loss=0.765875 avg=0.658994 VRAM=38.6GiB | 8.3% done | ETA(epoch): 928s + [Epoch 41][140/200] loss=0.609446 avg=0.660770 VRAM=38.5GiB | 8.3% done | ETA(epoch): 796s + [Epoch 41][150/200] loss=0.601211 avg=0.661177 VRAM=38.6GiB | 8.3% done | ETA(epoch): 663s + [Epoch 41][160/200] loss=0.704493 avg=0.662536 VRAM=38.5GiB | 8.4% done | ETA(epoch): 531s + [Epoch 41][170/200] loss=0.579069 avg=0.660634 VRAM=38.6GiB | 8.4% done | ETA(epoch): 398s + [Epoch 41][180/200] loss=0.753391 avg=0.659749 VRAM=38.5GiB | 8.4% done | ETA(epoch): 265s + [Epoch 41][190/200] loss=0.772961 avg=0.659109 VRAM=38.6GiB | 8.4% done | ETA(epoch): 133s + [Epoch 41][200/200] loss=0.640611 avg=0.658829 VRAM=38.5GiB | 8.4% done | ETA(epoch): 0s + Train loss: 0.658829 (2652.8s) ETA: 22516min + Val loss: 0.694222 [t_0.0-0.2=0.9377 t_0.2-0.4=0.7508 t_0.4-0.6=0.6411 t_0.6-0.8=0.5870 t_0.8-1.0=0.5882] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 42/499 (8% done) --- + [Epoch 42][10/200] loss=0.610599 avg=0.684614 VRAM=38.6GiB | 8.4% done | ETA(epoch): 2518s + [Epoch 42][20/200] loss=0.752055 avg=0.668647 VRAM=38.5GiB | 8.4% done | ETA(epoch): 2389s + [Epoch 42][30/200] loss=0.565358 avg=0.663969 VRAM=38.6GiB | 8.4% done | ETA(epoch): 2254s + [Epoch 42][40/200] loss=0.628694 avg=0.664068 VRAM=38.5GiB | 8.4% done | ETA(epoch): 2123s + [Epoch 42][50/200] loss=0.589997 avg=0.654072 VRAM=38.6GiB | 8.5% done | ETA(epoch): 1990s + [Epoch 42][60/200] loss=0.685161 avg=0.655164 VRAM=38.5GiB | 8.5% done | ETA(epoch): 1876s + [Epoch 42][70/200] loss=0.690750 avg=0.657461 VRAM=38.6GiB | 8.5% done | ETA(epoch): 1739s + [Epoch 42][80/200] loss=0.669014 avg=0.659324 VRAM=38.5GiB | 8.5% done | ETA(epoch): 1604s + [Epoch 42][90/200] loss=0.627863 avg=0.660235 VRAM=38.6GiB | 8.5% done | ETA(epoch): 1469s + [Epoch 42][100/200] loss=0.620488 avg=0.658559 VRAM=38.5GiB | 8.5% done | ETA(epoch): 1334s + [Epoch 42][110/200] loss=0.603792 avg=0.658522 VRAM=38.6GiB | 8.5% done | ETA(epoch): 1200s + [Epoch 42][120/200] loss=0.615483 avg=0.660562 VRAM=38.5GiB | 8.5% done | ETA(epoch): 1067s + [Epoch 42][130/200] loss=0.770257 avg=0.664343 VRAM=38.6GiB | 8.5% done | ETA(epoch): 933s + [Epoch 42][140/200] loss=0.697746 avg=0.664149 VRAM=38.5GiB | 8.5% done | ETA(epoch): 799s + [Epoch 42][150/200] loss=0.691090 avg=0.666506 VRAM=38.6GiB | 8.6% done | ETA(epoch): 667s + [Epoch 42][160/200] loss=0.613516 avg=0.665747 VRAM=38.5GiB | 8.6% done | ETA(epoch): 533s + [Epoch 42][170/200] loss=0.666035 avg=0.668952 VRAM=38.6GiB | 8.6% done | ETA(epoch): 400s + [Epoch 42][180/200] loss=0.634109 avg=0.669801 VRAM=38.5GiB | 8.6% done | ETA(epoch): 266s + [Epoch 42][190/200] loss=0.572006 avg=0.668715 VRAM=38.6GiB | 8.6% done | ETA(epoch): 133s + [Epoch 42][200/200] loss=0.524769 avg=0.668903 VRAM=38.5GiB | 8.6% done | ETA(epoch): 0s + Train loss: 0.668903 (2663.2s) ETA: 22460min + Val loss: 0.704393 [t_0.0-0.2=0.9235 t_0.2-0.4=0.7724 t_0.4-0.6=0.6468 t_0.6-0.8=0.5742 t_0.8-1.0=0.5977] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 43/499 (9% done) --- + [Epoch 43][10/200] loss=0.657274 avg=0.660997 VRAM=38.6GiB | 8.6% done | ETA(epoch): 2518s + [Epoch 43][20/200] loss=0.540158 avg=0.677397 VRAM=38.5GiB | 8.6% done | ETA(epoch): 2388s + [Epoch 43][30/200] loss=0.747491 avg=0.676172 VRAM=38.6GiB | 8.6% done | ETA(epoch): 2270s + [Epoch 43][40/200] loss=0.685127 avg=0.668702 VRAM=38.5GiB | 8.6% done | ETA(epoch): 2133s + [Epoch 43][50/200] loss=0.651846 avg=0.668250 VRAM=38.6GiB | 8.6% done | ETA(epoch): 1998s + [Epoch 43][60/200] loss=0.598152 avg=0.667200 VRAM=38.5GiB | 8.7% done | ETA(epoch): 1863s + [Epoch 43][70/200] loss=0.595171 avg=0.667255 VRAM=38.6GiB | 8.7% done | ETA(epoch): 1730s + [Epoch 43][80/200] loss=0.575660 avg=0.666252 VRAM=38.5GiB | 8.7% done | ETA(epoch): 1596s + [Epoch 43][90/200] loss=0.727694 avg=0.666409 VRAM=38.6GiB | 8.7% done | ETA(epoch): 1463s + [Epoch 43][100/200] loss=0.554138 avg=0.663426 VRAM=38.5GiB | 8.7% done | ETA(epoch): 1329s + [Epoch 43][110/200] loss=0.746068 avg=0.666911 VRAM=38.6GiB | 8.7% done | ETA(epoch): 1196s + [Epoch 43][120/200] loss=0.532632 avg=0.665865 VRAM=38.5GiB | 8.7% done | ETA(epoch): 1065s + [Epoch 43][130/200] loss=0.594197 avg=0.665157 VRAM=38.6GiB | 8.7% done | ETA(epoch): 932s + [Epoch 43][140/200] loss=0.584265 avg=0.665975 VRAM=38.5GiB | 8.7% done | ETA(epoch): 798s + [Epoch 43][150/200] loss=0.762793 avg=0.664008 VRAM=38.6GiB | 8.8% done | ETA(epoch): 665s + [Epoch 43][160/200] loss=0.640348 avg=0.666692 VRAM=38.5GiB | 8.8% done | ETA(epoch): 532s + [Epoch 43][170/200] loss=0.757885 avg=0.668062 VRAM=38.6GiB | 8.8% done | ETA(epoch): 399s + [Epoch 43][180/200] loss=0.655119 avg=0.667998 VRAM=38.5GiB | 8.8% done | ETA(epoch): 266s + [Epoch 43][190/200] loss=0.700205 avg=0.667560 VRAM=38.6GiB | 8.8% done | ETA(epoch): 133s + [Epoch 43][200/200] loss=0.498531 avg=0.668866 VRAM=38.5GiB | 8.8% done | ETA(epoch): 0s + Train loss: 0.668866 (2658.9s) ETA: 22404min + Val loss: 0.711646 [t_0.0-0.2=0.9574 t_0.2-0.4=0.7456 t_0.4-0.6=0.6564 t_0.6-0.8=0.5639 t_0.8-1.0=0.5968] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 20.5/188.5 GiB (10.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 44/499 (9% done) --- + [Epoch 44][10/200] loss=0.608053 avg=0.668947 VRAM=38.6GiB | 8.8% done | ETA(epoch): 2521s + [Epoch 44][20/200] loss=0.704137 avg=0.664798 VRAM=38.5GiB | 8.8% done | ETA(epoch): 2389s + [Epoch 44][30/200] loss=0.692834 avg=0.661673 VRAM=38.6GiB | 8.8% done | ETA(epoch): 2257s + [Epoch 44][40/200] loss=0.634531 avg=0.662140 VRAM=38.5GiB | 8.8% done | ETA(epoch): 2124s + [Epoch 44][50/200] loss=0.565670 avg=0.650725 VRAM=38.6GiB | 8.8% done | ETA(epoch): 1991s + [Epoch 44][60/200] loss=0.591946 avg=0.642988 VRAM=38.5GiB | 8.9% done | ETA(epoch): 1858s + [Epoch 44][70/200] loss=0.717716 avg=0.647458 VRAM=38.6GiB | 8.9% done | ETA(epoch): 1725s + [Epoch 44][80/200] loss=0.608360 avg=0.648854 VRAM=38.5GiB | 8.9% done | ETA(epoch): 1594s + [Epoch 44][90/200] loss=0.567967 avg=0.646066 VRAM=38.6GiB | 8.9% done | ETA(epoch): 1460s + [Epoch 44][100/200] loss=0.652819 avg=0.649611 VRAM=38.5GiB | 8.9% done | ETA(epoch): 1328s + [Epoch 44][110/200] loss=0.660363 avg=0.650284 VRAM=38.6GiB | 8.9% done | ETA(epoch): 1195s + [Epoch 44][120/200] loss=0.769061 avg=0.651862 VRAM=38.5GiB | 8.9% done | ETA(epoch): 1062s + [Epoch 44][130/200] loss=0.632576 avg=0.653131 VRAM=38.6GiB | 8.9% done | ETA(epoch): 929s + [Epoch 44][140/200] loss=0.794420 avg=0.656162 VRAM=38.5GiB | 8.9% done | ETA(epoch): 797s + [Epoch 44][150/200] loss=0.620079 avg=0.656200 VRAM=38.6GiB | 8.9% done | ETA(epoch): 664s + [Epoch 44][160/200] loss=0.655449 avg=0.655645 VRAM=38.5GiB | 9.0% done | ETA(epoch): 531s + [Epoch 44][170/200] loss=0.664742 avg=0.655695 VRAM=38.6GiB | 9.0% done | ETA(epoch): 398s + [Epoch 44][180/200] loss=0.720800 avg=0.660398 VRAM=38.5GiB | 9.0% done | ETA(epoch): 265s + [Epoch 44][190/200] loss=0.632587 avg=0.659351 VRAM=38.6GiB | 9.0% done | ETA(epoch): 133s + [Epoch 44][200/200] loss=0.593888 avg=0.657219 VRAM=38.5GiB | 9.0% done | ETA(epoch): 0s + Train loss: 0.657219 (2653.9s) ETA: 22347min + Val loss: 0.692430 [t_0.0-0.2=0.9173 t_0.2-0.4=0.7638 t_0.4-0.6=0.6362 t_0.6-0.8=0.5625 t_0.8-1.0=0.6034] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 45/499 (9% done) --- + [Epoch 45][10/200] loss=0.670697 avg=0.665118 VRAM=38.6GiB | 9.0% done | ETA(epoch): 2519s + [Epoch 45][20/200] loss=0.630285 avg=0.669366 VRAM=38.5GiB | 9.0% done | ETA(epoch): 2387s + [Epoch 45][30/200] loss=0.588332 avg=0.657108 VRAM=38.6GiB | 9.0% done | ETA(epoch): 2253s + [Epoch 45][40/200] loss=0.547578 avg=0.652757 VRAM=38.5GiB | 9.0% done | ETA(epoch): 2121s + [Epoch 45][50/200] loss=0.749766 avg=0.647821 VRAM=38.6GiB | 9.0% done | ETA(epoch): 1988s + [Epoch 45][60/200] loss=0.774795 avg=0.649944 VRAM=38.5GiB | 9.1% done | ETA(epoch): 1856s + [Epoch 45][70/200] loss=0.615941 avg=0.648099 VRAM=38.6GiB | 9.1% done | ETA(epoch): 1724s + [Epoch 45][80/200] loss=0.597544 avg=0.651931 VRAM=38.5GiB | 9.1% done | ETA(epoch): 1591s + [Epoch 45][90/200] loss=0.739690 avg=0.652843 VRAM=38.6GiB | 9.1% done | ETA(epoch): 1458s + [Epoch 45][100/200] loss=0.548056 avg=0.653274 VRAM=38.5GiB | 9.1% done | ETA(epoch): 1326s + [Epoch 45][110/200] loss=0.741462 avg=0.655376 VRAM=38.6GiB | 9.1% done | ETA(epoch): 1193s + [Epoch 45][120/200] loss=0.657792 avg=0.654681 VRAM=38.5GiB | 9.1% done | ETA(epoch): 1061s + [Epoch 45][130/200] loss=0.696133 avg=0.656125 VRAM=38.6GiB | 9.1% done | ETA(epoch): 929s + [Epoch 45][140/200] loss=0.607268 avg=0.652152 VRAM=38.5GiB | 9.1% done | ETA(epoch): 797s + [Epoch 45][150/200] loss=0.575948 avg=0.649827 VRAM=38.6GiB | 9.2% done | ETA(epoch): 664s + [Epoch 45][160/200] loss=0.726554 avg=0.650767 VRAM=38.5GiB | 9.2% done | ETA(epoch): 531s + [Epoch 45][170/200] loss=0.625359 avg=0.651681 VRAM=38.6GiB | 9.2% done | ETA(epoch): 398s + [Epoch 45][180/200] loss=0.564351 avg=0.649469 VRAM=38.5GiB | 9.2% done | ETA(epoch): 265s + [Epoch 45][190/200] loss=0.730802 avg=0.650748 VRAM=38.6GiB | 9.2% done | ETA(epoch): 133s + [Epoch 45][200/200] loss=0.733060 avg=0.650954 VRAM=38.5GiB | 9.2% done | ETA(epoch): 0s + Train loss: 0.650954 (2654.3s) ETA: 22291min + Val loss: 0.713147 [t_0.0-0.2=0.9696 t_0.2-0.4=0.7685 t_0.4-0.6=0.6373 t_0.6-0.8=0.5602 t_0.8-1.0=0.5750] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 46/499 (9% done) --- + [Epoch 46][10/200] loss=0.625065 avg=0.611890 VRAM=38.6GiB | 9.2% done | ETA(epoch): 2517s + [Epoch 46][20/200] loss=0.578455 avg=0.660116 VRAM=38.5GiB | 9.2% done | ETA(epoch): 2384s + [Epoch 46][30/200] loss=0.654104 avg=0.657900 VRAM=38.6GiB | 9.2% done | ETA(epoch): 2253s + [Epoch 46][40/200] loss=0.546653 avg=0.653713 VRAM=38.5GiB | 9.2% done | ETA(epoch): 2120s + [Epoch 46][50/200] loss=0.559917 avg=0.658573 VRAM=38.6GiB | 9.2% done | ETA(epoch): 1988s + [Epoch 46][60/200] loss=0.756923 avg=0.658815 VRAM=38.5GiB | 9.3% done | ETA(epoch): 1855s + [Epoch 46][70/200] loss=0.684667 avg=0.662917 VRAM=38.6GiB | 9.3% done | ETA(epoch): 1723s + [Epoch 46][80/200] loss=0.629322 avg=0.660414 VRAM=38.5GiB | 9.3% done | ETA(epoch): 1591s + [Epoch 46][90/200] loss=0.771343 avg=0.660060 VRAM=38.6GiB | 9.3% done | ETA(epoch): 1458s + [Epoch 46][100/200] loss=0.561416 avg=0.659959 VRAM=38.5GiB | 9.3% done | ETA(epoch): 1325s + [Epoch 46][110/200] loss=0.598429 avg=0.660315 VRAM=38.6GiB | 9.3% done | ETA(epoch): 1193s + [Epoch 46][120/200] loss=0.718965 avg=0.655752 VRAM=38.5GiB | 9.3% done | ETA(epoch): 1061s + [Epoch 46][130/200] loss=0.635753 avg=0.656762 VRAM=38.6GiB | 9.3% done | ETA(epoch): 928s + [Epoch 46][140/200] loss=0.854057 avg=0.659232 VRAM=38.5GiB | 9.3% done | ETA(epoch): 795s + [Epoch 46][150/200] loss=0.702610 avg=0.661694 VRAM=38.6GiB | 9.3% done | ETA(epoch): 663s + [Epoch 46][160/200] loss=0.595639 avg=0.661070 VRAM=38.5GiB | 9.4% done | ETA(epoch): 530s + [Epoch 46][170/200] loss=0.689114 avg=0.661225 VRAM=38.6GiB | 9.4% done | ETA(epoch): 398s + [Epoch 46][180/200] loss=0.613801 avg=0.659213 VRAM=38.5GiB | 9.4% done | ETA(epoch): 265s + [Epoch 46][190/200] loss=0.565491 avg=0.659774 VRAM=38.6GiB | 9.4% done | ETA(epoch): 133s + [Epoch 46][200/200] loss=0.663941 avg=0.657698 VRAM=38.5GiB | 9.4% done | ETA(epoch): 0s + Train loss: 0.657698 (2652.9s) ETA: 22235min + Val loss: 0.705871 [t_0.0-0.2=0.9577 t_0.2-0.4=0.7673 t_0.4-0.6=0.6320 t_0.6-0.8=0.5704 t_0.8-1.0=0.5916] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 47/499 (9% done) --- + [Epoch 47][10/200] loss=0.720074 avg=0.650461 VRAM=38.6GiB | 9.4% done | ETA(epoch): 2516s + [Epoch 47][20/200] loss=0.649502 avg=0.650842 VRAM=38.5GiB | 9.4% done | ETA(epoch): 2388s + [Epoch 47][30/200] loss=0.568427 avg=0.640460 VRAM=38.6GiB | 9.4% done | ETA(epoch): 2254s + [Epoch 47][40/200] loss=0.758773 avg=0.653462 VRAM=38.5GiB | 9.4% done | ETA(epoch): 2122s + [Epoch 47][50/200] loss=0.646442 avg=0.663501 VRAM=38.6GiB | 9.4% done | ETA(epoch): 1990s + [Epoch 47][60/200] loss=0.546076 avg=0.661354 VRAM=38.5GiB | 9.5% done | ETA(epoch): 1858s + [Epoch 47][70/200] loss=0.614831 avg=0.659530 VRAM=38.6GiB | 9.5% done | ETA(epoch): 1724s + [Epoch 47][80/200] loss=0.593321 avg=0.656191 VRAM=38.5GiB | 9.5% done | ETA(epoch): 1592s + [Epoch 47][90/200] loss=0.777521 avg=0.657698 VRAM=38.6GiB | 9.5% done | ETA(epoch): 1459s + [Epoch 47][100/200] loss=0.726053 avg=0.658900 VRAM=38.5GiB | 9.5% done | ETA(epoch): 1327s + [Epoch 47][110/200] loss=0.609037 avg=0.657887 VRAM=38.6GiB | 9.5% done | ETA(epoch): 1194s + [Epoch 47][120/200] loss=0.652225 avg=0.657180 VRAM=38.5GiB | 9.5% done | ETA(epoch): 1061s + [Epoch 47][130/200] loss=0.673794 avg=0.657475 VRAM=38.6GiB | 9.5% done | ETA(epoch): 929s + [Epoch 47][140/200] loss=0.646453 avg=0.658074 VRAM=38.5GiB | 9.5% done | ETA(epoch): 796s + [Epoch 47][150/200] loss=0.632706 avg=0.657469 VRAM=38.6GiB | 9.6% done | ETA(epoch): 665s + [Epoch 47][160/200] loss=0.707334 avg=0.657555 VRAM=38.5GiB | 9.6% done | ETA(epoch): 532s + [Epoch 47][170/200] loss=0.653602 avg=0.658762 VRAM=38.6GiB | 9.6% done | ETA(epoch): 399s + [Epoch 47][180/200] loss=0.548835 avg=0.658847 VRAM=38.5GiB | 9.6% done | ETA(epoch): 266s + [Epoch 47][190/200] loss=0.745228 avg=0.660169 VRAM=38.6GiB | 9.6% done | ETA(epoch): 133s + [Epoch 47][200/200] loss=0.645841 avg=0.660770 VRAM=38.5GiB | 9.6% done | ETA(epoch): 0s + Train loss: 0.660770 (2658.1s) ETA: 22180min + Val loss: 0.706602 [t_0.0-0.2=0.9583 t_0.2-0.4=0.7643 t_0.4-0.6=0.6427 t_0.6-0.8=0.5741 t_0.8-1.0=0.5809] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 48/499 (10% done) --- + [Epoch 48][10/200] loss=0.816022 avg=0.723032 VRAM=38.6GiB | 9.6% done | ETA(epoch): 2520s + [Epoch 48][20/200] loss=0.569560 avg=0.683381 VRAM=38.5GiB | 9.6% done | ETA(epoch): 2390s + [Epoch 48][30/200] loss=0.707335 avg=0.671961 VRAM=38.6GiB | 9.6% done | ETA(epoch): 2257s + [Epoch 48][40/200] loss=0.591247 avg=0.665867 VRAM=38.5GiB | 9.6% done | ETA(epoch): 2124s + [Epoch 48][50/200] loss=0.573742 avg=0.668018 VRAM=38.6GiB | 9.7% done | ETA(epoch): 1991s + [Epoch 48][60/200] loss=0.635042 avg=0.672803 VRAM=38.5GiB | 9.7% done | ETA(epoch): 1858s + [Epoch 48][70/200] loss=0.677004 avg=0.671912 VRAM=38.6GiB | 9.7% done | ETA(epoch): 1725s + [Epoch 48][80/200] loss=0.580813 avg=0.665219 VRAM=38.5GiB | 9.7% done | ETA(epoch): 1593s + [Epoch 48][90/200] loss=0.703009 avg=0.664411 VRAM=38.6GiB | 9.7% done | ETA(epoch): 1460s + [Epoch 48][100/200] loss=0.589639 avg=0.662860 VRAM=38.5GiB | 9.7% done | ETA(epoch): 1327s + [Epoch 48][110/200] loss=0.687400 avg=0.663610 VRAM=38.6GiB | 9.7% done | ETA(epoch): 1195s + [Epoch 48][120/200] loss=0.742469 avg=0.662894 VRAM=38.5GiB | 9.7% done | ETA(epoch): 1062s + [Epoch 48][130/200] loss=0.560421 avg=0.663993 VRAM=38.6GiB | 9.7% done | ETA(epoch): 929s + [Epoch 48][140/200] loss=0.745566 avg=0.664241 VRAM=38.5GiB | 9.7% done | ETA(epoch): 796s + [Epoch 48][150/200] loss=0.627082 avg=0.661614 VRAM=38.6GiB | 9.8% done | ETA(epoch): 664s + [Epoch 48][160/200] loss=0.771876 avg=0.663396 VRAM=38.5GiB | 9.8% done | ETA(epoch): 531s + [Epoch 48][170/200] loss=0.622411 avg=0.663574 VRAM=38.6GiB | 9.8% done | ETA(epoch): 398s + [Epoch 48][180/200] loss=0.605466 avg=0.664163 VRAM=38.5GiB | 9.8% done | ETA(epoch): 265s + [Epoch 48][190/200] loss=0.693597 avg=0.662607 VRAM=38.6GiB | 9.8% done | ETA(epoch): 133s + [Epoch 48][200/200] loss=0.538998 avg=0.661643 VRAM=38.5GiB | 9.8% done | ETA(epoch): 0s + Train loss: 0.661643 (2658.6s) ETA: 22125min + Val loss: 0.698196 [t_0.0-0.2=0.9322 t_0.2-0.4=0.7500 t_0.4-0.6=0.6337 t_0.6-0.8=0.5639 t_0.8-1.0=0.5773] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 20.6/188.5 GiB (10.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 49/499 (10% done) --- + [Epoch 49][10/200] loss=0.681949 avg=0.690092 VRAM=38.6GiB | 9.8% done | ETA(epoch): 2523s + [Epoch 49][20/200] loss=0.594839 avg=0.696830 VRAM=38.5GiB | 9.8% done | ETA(epoch): 2387s + [Epoch 49][30/200] loss=0.597435 avg=0.692864 VRAM=38.6GiB | 9.8% done | ETA(epoch): 2256s + [Epoch 49][40/200] loss=0.554655 avg=0.682995 VRAM=38.5GiB | 9.8% done | ETA(epoch): 2122s + [Epoch 49][50/200] loss=0.664976 avg=0.671734 VRAM=38.6GiB | 9.8% done | ETA(epoch): 1990s + [Epoch 49][60/200] loss=0.631623 avg=0.671099 VRAM=38.5GiB | 9.9% done | ETA(epoch): 1858s + [Epoch 49][70/200] loss=0.671238 avg=0.672736 VRAM=38.6GiB | 9.9% done | ETA(epoch): 1726s + [Epoch 49][80/200] loss=0.637248 avg=0.670759 VRAM=38.5GiB | 9.9% done | ETA(epoch): 1593s + [Epoch 49][90/200] loss=0.575709 avg=0.668160 VRAM=38.6GiB | 9.9% done | ETA(epoch): 1460s + [Epoch 49][100/200] loss=0.728615 avg=0.671608 VRAM=38.5GiB | 9.9% done | ETA(epoch): 1327s + [Epoch 49][110/200] loss=0.612114 avg=0.670113 VRAM=38.6GiB | 9.9% done | ETA(epoch): 1194s + [Epoch 49][120/200] loss=0.658670 avg=0.668477 VRAM=38.5GiB | 9.9% done | ETA(epoch): 1062s + [Epoch 49][130/200] loss=0.687695 avg=0.669397 VRAM=38.6GiB | 9.9% done | ETA(epoch): 929s + [Epoch 49][140/200] loss=0.610700 avg=0.666952 VRAM=38.5GiB | 9.9% done | ETA(epoch): 796s + [Epoch 49][150/200] loss=0.614570 avg=0.663385 VRAM=38.6GiB | 10.0% done | ETA(epoch): 664s + [Epoch 49][160/200] loss=0.659108 avg=0.662591 VRAM=38.5GiB | 10.0% done | ETA(epoch): 532s + [Epoch 49][170/200] loss=0.668690 avg=0.661509 VRAM=38.6GiB | 10.0% done | ETA(epoch): 399s + [Epoch 49][180/200] loss=0.623640 avg=0.661961 VRAM=38.5GiB | 10.0% done | ETA(epoch): 266s + [Epoch 49][190/200] loss=0.672176 avg=0.661550 VRAM=38.6GiB | 10.0% done | ETA(epoch): 133s + [Epoch 49][200/200] loss=0.837823 avg=0.663320 VRAM=38.5GiB | 10.0% done | ETA(epoch): 0s + Train loss: 0.663320 (2659.6s) ETA: 22071min + Val loss: 0.701741 [t_0.0-0.2=0.9569 t_0.2-0.4=0.7555 t_0.4-0.6=0.6419 t_0.6-0.8=0.5676 t_0.8-1.0=0.5965] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 50/499 (10% done) --- + [MilestoneVis] train_0 step 10000 ✓ + [MilestoneGrid] train_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 10000 ✓ + [MilestoneGrid] train_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 10000 ✓ + [MilestoneGrid] val_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 10000 ✓ + [MilestoneGrid] val_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 10000 ✓ + [MilestoneGrid] test_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 10000 ✓ + [MilestoneGrid] test_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 10000 ✓ + [MilestoneGrid] test_2 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 10000 ✓ + [MilestoneGrid] test_3 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 10000 ✓ + [MilestoneGrid] test_4 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 10000 ✓ + [MilestoneGrid] test_5 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_5.png + [MilestoneVis] test_6 step 10000 ✓ + [MilestoneGrid] test_6 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_6.png + [MilestoneVis] test_7 step 10000 ✓ + [MilestoneGrid] test_7 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_7.png + [MilestoneVis] test_8 step 10000 ✓ + [MilestoneGrid] test_8 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_8.png + [MilestoneVis] test_9 step 10000 ✓ + [MilestoneGrid] test_9 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_9.png + [MilestoneVis] test_10 step 10000 ✓ + [MilestoneGrid] test_10 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_10.png + [MilestoneVis] test_11 step 10000 ✓ + [MilestoneGrid] test_11 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_11.png + [MilestoneVis] test_12 step 10000 ✓ + [MilestoneGrid] test_12 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_12.png + [MilestoneVis] test_13 step 10000 ✓ + [MilestoneGrid] test_13 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_13.png + [MilestoneVis] test_14 step 10000 ✓ + [MilestoneGrid] test_14 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_14.png + [MilestoneVis] test_15 step 10000 ✓ + [MilestoneGrid] test_15 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_15.png + [MilestoneVis] test_16 step 10000 ✓ + [MilestoneGrid] test_16 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_16.png + [Epoch 50][10/200] loss=0.642215 avg=0.683374 VRAM=38.6GiB | 10.0% done | ETA(epoch): 43956s + [Epoch 50][20/200] loss=0.724113 avg=0.664929 VRAM=38.5GiB | 10.0% done | ETA(epoch): 22014s + [Epoch 50][30/200] loss=0.701347 avg=0.665394 VRAM=38.6GiB | 10.0% done | ETA(epoch): 14613s + [Epoch 50][40/200] loss=0.683647 avg=0.669357 VRAM=38.5GiB | 10.0% done | ETA(epoch): 10844s + [Epoch 50][50/200] loss=0.611302 avg=0.665802 VRAM=38.6GiB | 10.1% done | ETA(epoch): 8550s + [Epoch 50][60/200] loss=0.655980 avg=0.669922 VRAM=38.5GiB | 10.1% done | ETA(epoch): 6959s + [Epoch 50][70/200] loss=0.696212 avg=0.667799 VRAM=38.6GiB | 10.1% done | ETA(epoch): 5785s + [Epoch 50][80/200] loss=0.612502 avg=0.664492 VRAM=38.5GiB | 10.1% done | ETA(epoch): 4871s + [Epoch 50][90/200] loss=0.734926 avg=0.669060 VRAM=38.6GiB | 10.1% done | ETA(epoch): 4131s + [Epoch 50][100/200] loss=0.573832 avg=0.667684 VRAM=38.5GiB | 10.1% done | ETA(epoch): 3512s + [Epoch 50][110/200] loss=0.665197 avg=0.665257 VRAM=38.6GiB | 10.1% done | ETA(epoch): 2982s + [Epoch 50][120/200] loss=0.615893 avg=0.666353 VRAM=38.5GiB | 10.1% done | ETA(epoch): 2518s + [Epoch 50][130/200] loss=0.749259 avg=0.666570 VRAM=38.6GiB | 10.1% done | ETA(epoch): 2106s + [Epoch 50][140/200] loss=0.662844 avg=0.665977 VRAM=38.5GiB | 10.1% done | ETA(epoch): 1733s + [Epoch 50][150/200] loss=0.617167 avg=0.665964 VRAM=38.6GiB | 10.2% done | ETA(epoch): 1393s + [Epoch 50][160/200] loss=0.742345 avg=0.662752 VRAM=38.5GiB | 10.2% done | ETA(epoch): 1078s + [Epoch 50][170/200] loss=0.640262 avg=0.662396 VRAM=38.6GiB | 10.2% done | ETA(epoch): 784s + [Epoch 50][180/200] loss=0.619639 avg=0.660436 VRAM=38.5GiB | 10.2% done | ETA(epoch): 508s + [Epoch 50][190/200] loss=0.648112 avg=0.661137 VRAM=38.6GiB | 10.2% done | ETA(epoch): 248s + [Epoch 50][200/200] loss=0.622784 avg=0.662231 VRAM=38.5GiB | 10.2% done | ETA(epoch): 0s + Train loss: 0.662231 (4841.1s) ETA: 22337min + Val loss: 0.688562 [t_0.0-0.2=0.9297 t_0.2-0.4=0.7481 t_0.4-0.6=0.6611 t_0.6-0.8=0.5720 t_0.8-1.0=0.5706] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 51/499 (10% done) --- + [Epoch 51][10/200] loss=0.560633 avg=0.640487 VRAM=38.6GiB | 10.2% done | ETA(epoch): 2515s + [Epoch 51][20/200] loss=0.647975 avg=0.645528 VRAM=38.5GiB | 10.2% done | ETA(epoch): 2407s + [Epoch 51][30/200] loss=0.562811 avg=0.655087 VRAM=38.6GiB | 10.2% done | ETA(epoch): 2266s + [Epoch 51][40/200] loss=0.573938 avg=0.649639 VRAM=38.5GiB | 10.2% done | ETA(epoch): 2130s + [Epoch 51][50/200] loss=0.623218 avg=0.648891 VRAM=38.6GiB | 10.2% done | ETA(epoch): 1994s + [Epoch 51][60/200] loss=0.750261 avg=0.649290 VRAM=38.5GiB | 10.3% done | ETA(epoch): 1860s + [Epoch 51][70/200] loss=0.776723 avg=0.656223 VRAM=38.6GiB | 10.3% done | ETA(epoch): 1727s + [Epoch 51][80/200] loss=0.693393 avg=0.658342 VRAM=38.5GiB | 10.3% done | ETA(epoch): 1594s + [Epoch 51][90/200] loss=0.767339 avg=0.663706 VRAM=38.6GiB | 10.3% done | ETA(epoch): 1461s + [Epoch 51][100/200] loss=0.714996 avg=0.666283 VRAM=38.5GiB | 10.3% done | ETA(epoch): 1328s + [Epoch 51][110/200] loss=0.686287 avg=0.665561 VRAM=38.6GiB | 10.3% done | ETA(epoch): 1195s + [Epoch 51][120/200] loss=0.606537 avg=0.663828 VRAM=38.5GiB | 10.3% done | ETA(epoch): 1062s + [Epoch 51][130/200] loss=0.559713 avg=0.661831 VRAM=38.6GiB | 10.3% done | ETA(epoch): 929s + [Epoch 51][140/200] loss=0.682167 avg=0.661155 VRAM=38.5GiB | 10.3% done | ETA(epoch): 796s + [Epoch 51][150/200] loss=0.550845 avg=0.662888 VRAM=38.6GiB | 10.3% done | ETA(epoch): 664s + [Epoch 51][160/200] loss=0.670614 avg=0.663301 VRAM=38.5GiB | 10.4% done | ETA(epoch): 531s + [Epoch 51][170/200] loss=0.708798 avg=0.663383 VRAM=38.6GiB | 10.4% done | ETA(epoch): 398s + [Epoch 51][180/200] loss=0.795892 avg=0.664616 VRAM=38.5GiB | 10.4% done | ETA(epoch): 265s + [Epoch 51][190/200] loss=0.617776 avg=0.663814 VRAM=38.6GiB | 10.4% done | ETA(epoch): 133s + [Epoch 51][200/200] loss=0.702393 avg=0.666098 VRAM=38.5GiB | 10.4% done | ETA(epoch): 0s + Train loss: 0.666098 (2653.6s) ETA: 22276min + Val loss: 0.694545 [t_0.0-0.2=0.9249 t_0.2-0.4=0.7606 t_0.4-0.6=0.6400 t_0.6-0.8=0.5662 t_0.8-1.0=0.5714] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 52/499 (10% done) --- + [Epoch 52][10/200] loss=0.623544 avg=0.639243 VRAM=38.6GiB | 10.4% done | ETA(epoch): 2522s + [Epoch 52][20/200] loss=0.637169 avg=0.640492 VRAM=38.5GiB | 10.4% done | ETA(epoch): 2388s + [Epoch 52][30/200] loss=0.630050 avg=0.640488 VRAM=38.6GiB | 10.4% done | ETA(epoch): 2256s + [Epoch 52][40/200] loss=0.598106 avg=0.644529 VRAM=38.5GiB | 10.4% done | ETA(epoch): 2123s + [Epoch 52][50/200] loss=0.684466 avg=0.649863 VRAM=38.6GiB | 10.4% done | ETA(epoch): 1990s + [Epoch 52][60/200] loss=0.610645 avg=0.650495 VRAM=38.5GiB | 10.5% done | ETA(epoch): 1857s + [Epoch 52][70/200] loss=0.589426 avg=0.653828 VRAM=38.6GiB | 10.5% done | ETA(epoch): 1724s + [Epoch 52][80/200] loss=0.548087 avg=0.649140 VRAM=38.5GiB | 10.5% done | ETA(epoch): 1592s + [Epoch 52][90/200] loss=0.696545 avg=0.652402 VRAM=38.6GiB | 10.5% done | ETA(epoch): 1459s + [Epoch 52][100/200] loss=0.675048 avg=0.652231 VRAM=38.5GiB | 10.5% done | ETA(epoch): 1326s + [Epoch 52][110/200] loss=0.746947 avg=0.652610 VRAM=38.6GiB | 10.5% done | ETA(epoch): 1194s + [Epoch 52][120/200] loss=0.777583 avg=0.654310 VRAM=38.5GiB | 10.5% done | ETA(epoch): 1061s + [Epoch 52][130/200] loss=0.573151 avg=0.651354 VRAM=38.6GiB | 10.5% done | ETA(epoch): 928s + [Epoch 52][140/200] loss=0.636187 avg=0.651619 VRAM=38.5GiB | 10.5% done | ETA(epoch): 796s + [Epoch 52][150/200] loss=0.663815 avg=0.655002 VRAM=38.6GiB | 10.5% done | ETA(epoch): 663s + [Epoch 52][160/200] loss=0.584301 avg=0.653259 VRAM=38.5GiB | 10.6% done | ETA(epoch): 531s + [Epoch 52][170/200] loss=0.822681 avg=0.652604 VRAM=38.6GiB | 10.6% done | ETA(epoch): 398s + [Epoch 52][180/200] loss=0.556110 avg=0.651787 VRAM=38.5GiB | 10.6% done | ETA(epoch): 265s + [Epoch 52][190/200] loss=0.547683 avg=0.651682 VRAM=38.6GiB | 10.6% done | ETA(epoch): 133s + [Epoch 52][200/200] loss=0.691292 avg=0.652757 VRAM=38.5GiB | 10.6% done | ETA(epoch): 0s + Train loss: 0.652757 (2652.8s) ETA: 22214min + Val loss: 0.684917 [t_0.0-0.2=0.9387 t_0.2-0.4=0.7521 t_0.4-0.6=0.6481 t_0.6-0.8=0.5632 t_0.8-1.0=0.5932] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0052 (BEST) + Deleted old checkpoint: checkpoint_epoch_0014 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 53/499 (11% done) --- + [Epoch 53][10/200] loss=0.631178 avg=0.675450 VRAM=38.6GiB | 10.6% done | ETA(epoch): 2521s + [Epoch 53][20/200] loss=0.611234 avg=0.671230 VRAM=38.5GiB | 10.6% done | ETA(epoch): 2386s + [Epoch 53][30/200] loss=0.692215 avg=0.682635 VRAM=38.6GiB | 10.6% done | ETA(epoch): 2274s + [Epoch 53][40/200] loss=0.647543 avg=0.674281 VRAM=38.5GiB | 10.6% done | ETA(epoch): 2136s + [Epoch 53][50/200] loss=0.720782 avg=0.675284 VRAM=38.6GiB | 10.7% done | ETA(epoch): 2000s + [Epoch 53][60/200] loss=0.612936 avg=0.670977 VRAM=38.5GiB | 10.7% done | ETA(epoch): 1865s + [Epoch 53][70/200] loss=0.612971 avg=0.667820 VRAM=38.6GiB | 10.7% done | ETA(epoch): 1731s + [Epoch 53][80/200] loss=0.646590 avg=0.669611 VRAM=38.5GiB | 10.7% done | ETA(epoch): 1597s + [Epoch 53][90/200] loss=0.646213 avg=0.668294 VRAM=38.6GiB | 10.7% done | ETA(epoch): 1463s + [Epoch 53][100/200] loss=0.700643 avg=0.667907 VRAM=38.5GiB | 10.7% done | ETA(epoch): 1330s + [Epoch 53][110/200] loss=0.612434 avg=0.669821 VRAM=38.6GiB | 10.7% done | ETA(epoch): 1197s + [Epoch 53][120/200] loss=0.627732 avg=0.668110 VRAM=38.5GiB | 10.7% done | ETA(epoch): 1063s + [Epoch 53][130/200] loss=0.543258 avg=0.665122 VRAM=38.6GiB | 10.7% done | ETA(epoch): 930s + [Epoch 53][140/200] loss=0.681349 avg=0.670031 VRAM=38.5GiB | 10.7% done | ETA(epoch): 797s + [Epoch 53][150/200] loss=0.580852 avg=0.668504 VRAM=38.6GiB | 10.8% done | ETA(epoch): 664s + [Epoch 53][160/200] loss=0.608464 avg=0.667020 VRAM=38.5GiB | 10.8% done | ETA(epoch): 531s + [Epoch 53][170/200] loss=0.637832 avg=0.664481 VRAM=38.6GiB | 10.8% done | ETA(epoch): 398s + [Epoch 53][180/200] loss=0.645980 avg=0.663049 VRAM=38.5GiB | 10.8% done | ETA(epoch): 266s + [Epoch 53][190/200] loss=0.772427 avg=0.664056 VRAM=38.6GiB | 10.8% done | ETA(epoch): 133s + [Epoch 53][200/200] loss=0.680075 avg=0.664051 VRAM=38.5GiB | 10.8% done | ETA(epoch): 0s + Train loss: 0.664051 (2655.5s) ETA: 22154min + Val loss: 0.681645 [t_0.0-0.2=0.9206 t_0.2-0.4=0.7523 t_0.4-0.6=0.6572 t_0.6-0.8=0.5641 t_0.8-1.0=0.5969] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0053 (BEST) + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 54/499 (11% done) --- + [Epoch 54][10/200] loss=0.755654 avg=0.680922 VRAM=38.6GiB | 10.8% done | ETA(epoch): 2523s + [Epoch 54][20/200] loss=0.719464 avg=0.658351 VRAM=38.5GiB | 10.8% done | ETA(epoch): 2387s + [Epoch 54][30/200] loss=0.694952 avg=0.662187 VRAM=38.6GiB | 10.8% done | ETA(epoch): 2255s + [Epoch 54][40/200] loss=0.702876 avg=0.663199 VRAM=38.5GiB | 10.8% done | ETA(epoch): 2121s + [Epoch 54][50/200] loss=0.796894 avg=0.672738 VRAM=38.6GiB | 10.8% done | ETA(epoch): 1989s + [Epoch 54][60/200] loss=0.743029 avg=0.668582 VRAM=38.5GiB | 10.9% done | ETA(epoch): 1856s + [Epoch 54][70/200] loss=0.680959 avg=0.670253 VRAM=38.6GiB | 10.9% done | ETA(epoch): 1724s + [Epoch 54][80/200] loss=0.551626 avg=0.665529 VRAM=38.5GiB | 10.9% done | ETA(epoch): 1591s + [Epoch 54][90/200] loss=0.705545 avg=0.662021 VRAM=38.6GiB | 10.9% done | ETA(epoch): 1462s + [Epoch 54][100/200] loss=0.603232 avg=0.661983 VRAM=38.5GiB | 10.9% done | ETA(epoch): 1329s + [Epoch 54][110/200] loss=0.561553 avg=0.660898 VRAM=38.6GiB | 10.9% done | ETA(epoch): 1196s + [Epoch 54][120/200] loss=0.667626 avg=0.656396 VRAM=38.5GiB | 10.9% done | ETA(epoch): 1062s + [Epoch 54][130/200] loss=0.798250 avg=0.660819 VRAM=38.6GiB | 10.9% done | ETA(epoch): 930s + [Epoch 54][140/200] loss=0.734235 avg=0.660231 VRAM=38.5GiB | 10.9% done | ETA(epoch): 797s + [Epoch 54][150/200] loss=0.686550 avg=0.660848 VRAM=38.6GiB | 10.9% done | ETA(epoch): 664s + [Epoch 54][160/200] loss=0.688719 avg=0.660820 VRAM=38.5GiB | 11.0% done | ETA(epoch): 531s + [Epoch 54][170/200] loss=0.672535 avg=0.661015 VRAM=38.6GiB | 11.0% done | ETA(epoch): 398s + [Epoch 54][180/200] loss=0.686123 avg=0.661385 VRAM=38.5GiB | 11.0% done | ETA(epoch): 266s + [Epoch 54][190/200] loss=0.634691 avg=0.660017 VRAM=38.6GiB | 11.0% done | ETA(epoch): 133s + [Epoch 54][200/200] loss=0.758178 avg=0.660097 VRAM=38.5GiB | 11.0% done | ETA(epoch): 0s + Train loss: 0.660097 (2655.1s) ETA: 22094min + Val loss: 0.705747 [t_0.0-0.2=0.9510 t_0.2-0.4=0.7658 t_0.4-0.6=0.6257 t_0.6-0.8=0.5562 t_0.8-1.0=0.5979] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 22.1/188.5 GiB (11.7%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 55/499 (11% done) --- + [Epoch 55][10/200] loss=0.569679 avg=0.668359 VRAM=38.6GiB | 11.0% done | ETA(epoch): 2521s + [Epoch 55][20/200] loss=0.652188 avg=0.664946 VRAM=38.5GiB | 11.0% done | ETA(epoch): 2387s + [Epoch 55][30/200] loss=0.649170 avg=0.672216 VRAM=38.6GiB | 11.0% done | ETA(epoch): 2255s + [Epoch 55][40/200] loss=0.691630 avg=0.676269 VRAM=38.5GiB | 11.0% done | ETA(epoch): 2121s + [Epoch 55][50/200] loss=0.715587 avg=0.674096 VRAM=38.6GiB | 11.1% done | ETA(epoch): 1998s + [Epoch 55][60/200] loss=0.666667 avg=0.674665 VRAM=38.5GiB | 11.1% done | ETA(epoch): 1864s + [Epoch 55][70/200] loss=0.675501 avg=0.675148 VRAM=38.6GiB | 11.1% done | ETA(epoch): 1729s + [Epoch 55][80/200] loss=0.663429 avg=0.675047 VRAM=38.5GiB | 11.1% done | ETA(epoch): 1596s + [Epoch 55][90/200] loss=0.614557 avg=0.672085 VRAM=38.6GiB | 11.1% done | ETA(epoch): 1463s + [Epoch 55][100/200] loss=0.694979 avg=0.670821 VRAM=38.5GiB | 11.1% done | ETA(epoch): 1329s + [Epoch 55][110/200] loss=0.673942 avg=0.667965 VRAM=38.6GiB | 11.1% done | ETA(epoch): 1196s + [Epoch 55][120/200] loss=0.602374 avg=0.666835 VRAM=38.5GiB | 11.1% done | ETA(epoch): 1063s + [Epoch 55][130/200] loss=0.529391 avg=0.665253 VRAM=38.6GiB | 11.1% done | ETA(epoch): 930s + [Epoch 55][140/200] loss=0.569753 avg=0.666430 VRAM=38.5GiB | 11.1% done | ETA(epoch): 797s + [Epoch 55][150/200] loss=0.699301 avg=0.668451 VRAM=38.6GiB | 11.2% done | ETA(epoch): 664s + [Epoch 55][160/200] loss=0.617451 avg=0.666481 VRAM=38.5GiB | 11.2% done | ETA(epoch): 531s + [Epoch 55][170/200] loss=0.592322 avg=0.666258 VRAM=38.6GiB | 11.2% done | ETA(epoch): 399s + [Epoch 55][180/200] loss=0.615526 avg=0.668010 VRAM=38.5GiB | 11.2% done | ETA(epoch): 266s + [Epoch 55][190/200] loss=0.706809 avg=0.665960 VRAM=38.6GiB | 11.2% done | ETA(epoch): 133s + [Epoch 55][200/200] loss=0.686514 avg=0.665123 VRAM=38.5GiB | 11.2% done | ETA(epoch): 0s + Train loss: 0.665123 (2656.5s) ETA: 22035min + Val loss: 0.717732 [t_0.0-0.2=0.9326 t_0.2-0.4=0.7725 t_0.4-0.6=0.6507 t_0.6-0.8=0.5516 t_0.8-1.0=0.5898] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 56/499 (11% done) --- + [Epoch 56][10/200] loss=0.589217 avg=0.632103 VRAM=38.6GiB | 11.2% done | ETA(epoch): 2568s + [Epoch 56][20/200] loss=0.652894 avg=0.649823 VRAM=38.5GiB | 11.2% done | ETA(epoch): 2410s + [Epoch 56][30/200] loss=0.662236 avg=0.657821 VRAM=38.6GiB | 11.2% done | ETA(epoch): 2269s + [Epoch 56][40/200] loss=0.643804 avg=0.657668 VRAM=38.5GiB | 11.2% done | ETA(epoch): 2132s + [Epoch 56][50/200] loss=0.643342 avg=0.663686 VRAM=38.6GiB | 11.2% done | ETA(epoch): 1997s + [Epoch 56][60/200] loss=0.550377 avg=0.656131 VRAM=38.5GiB | 11.3% done | ETA(epoch): 1862s + [Epoch 56][70/200] loss=0.704356 avg=0.660841 VRAM=38.6GiB | 11.3% done | ETA(epoch): 1729s + [Epoch 56][80/200] loss=0.650514 avg=0.660817 VRAM=38.5GiB | 11.3% done | ETA(epoch): 1596s + [Epoch 56][90/200] loss=0.732383 avg=0.660664 VRAM=38.6GiB | 11.3% done | ETA(epoch): 1462s + [Epoch 56][100/200] loss=0.668644 avg=0.659179 VRAM=38.5GiB | 11.3% done | ETA(epoch): 1332s + [Epoch 56][110/200] loss=0.752346 avg=0.660776 VRAM=38.6GiB | 11.3% done | ETA(epoch): 1199s + [Epoch 56][120/200] loss=0.602249 avg=0.657666 VRAM=38.5GiB | 11.3% done | ETA(epoch): 1065s + [Epoch 56][130/200] loss=0.867067 avg=0.659083 VRAM=38.6GiB | 11.3% done | ETA(epoch): 932s + [Epoch 56][140/200] loss=0.731140 avg=0.659382 VRAM=38.5GiB | 11.3% done | ETA(epoch): 798s + [Epoch 56][150/200] loss=0.652443 avg=0.657915 VRAM=38.6GiB | 11.3% done | ETA(epoch): 665s + [Epoch 56][160/200] loss=0.547295 avg=0.657547 VRAM=38.5GiB | 11.4% done | ETA(epoch): 532s + [Epoch 56][170/200] loss=0.583605 avg=0.656561 VRAM=38.6GiB | 11.4% done | ETA(epoch): 399s + [Epoch 56][180/200] loss=0.689401 avg=0.659173 VRAM=38.5GiB | 11.4% done | ETA(epoch): 266s + [Epoch 56][190/200] loss=0.735749 avg=0.660958 VRAM=38.6GiB | 11.4% done | ETA(epoch): 133s + [Epoch 56][200/200] loss=0.584537 avg=0.660355 VRAM=38.5GiB | 11.4% done | ETA(epoch): 0s + Train loss: 0.660355 (2658.9s) ETA: 21976min + Val loss: 0.683918 [t_0.0-0.2=0.9344 t_0.2-0.4=0.7562 t_0.4-0.6=0.6500 t_0.6-0.8=0.5642 t_0.8-1.0=0.5954] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0056 +[MEM @ epoch 56 end] RAM: 23.5/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 57/499 (11% done) --- + [Epoch 57][10/200] loss=0.863389 avg=0.667510 VRAM=38.6GiB | 11.4% done | ETA(epoch): 2525s + [Epoch 57][20/200] loss=0.596902 avg=0.660309 VRAM=38.5GiB | 11.4% done | ETA(epoch): 2388s + [Epoch 57][30/200] loss=0.580249 avg=0.653848 VRAM=38.6GiB | 11.4% done | ETA(epoch): 2256s + [Epoch 57][40/200] loss=0.607084 avg=0.655849 VRAM=38.5GiB | 11.4% done | ETA(epoch): 2123s + [Epoch 57][50/200] loss=0.821364 avg=0.654508 VRAM=38.6GiB | 11.5% done | ETA(epoch): 1991s + [Epoch 57][60/200] loss=0.599297 avg=0.649935 VRAM=38.5GiB | 11.5% done | ETA(epoch): 1858s + [Epoch 57][70/200] loss=0.663999 avg=0.650999 VRAM=38.6GiB | 11.5% done | ETA(epoch): 1725s + [Epoch 57][80/200] loss=0.690987 avg=0.656166 VRAM=38.5GiB | 11.5% done | ETA(epoch): 1592s + [Epoch 57][90/200] loss=0.663427 avg=0.656653 VRAM=38.6GiB | 11.5% done | ETA(epoch): 1460s + [Epoch 57][100/200] loss=0.515944 avg=0.656024 VRAM=38.5GiB | 11.5% done | ETA(epoch): 1327s + [Epoch 57][110/200] loss=0.586462 avg=0.653047 VRAM=38.6GiB | 11.5% done | ETA(epoch): 1194s + [Epoch 57][120/200] loss=0.638013 avg=0.654767 VRAM=38.5GiB | 11.5% done | ETA(epoch): 1062s + [Epoch 57][130/200] loss=0.630141 avg=0.652053 VRAM=38.6GiB | 11.5% done | ETA(epoch): 929s + [Epoch 57][140/200] loss=0.738549 avg=0.654525 VRAM=38.5GiB | 11.5% done | ETA(epoch): 796s + [Epoch 57][150/200] loss=0.637698 avg=0.652706 VRAM=38.6GiB | 11.6% done | ETA(epoch): 663s + [Epoch 57][160/200] loss=0.628317 avg=0.654408 VRAM=38.5GiB | 11.6% done | ETA(epoch): 532s + [Epoch 57][170/200] loss=0.617642 avg=0.653526 VRAM=38.6GiB | 11.6% done | ETA(epoch): 399s + [Epoch 57][180/200] loss=0.583384 avg=0.653770 VRAM=38.5GiB | 11.6% done | ETA(epoch): 266s + [Epoch 57][190/200] loss=0.685221 avg=0.653673 VRAM=38.6GiB | 11.6% done | ETA(epoch): 133s + [Epoch 57][200/200] loss=0.685259 avg=0.653061 VRAM=38.5GiB | 11.6% done | ETA(epoch): 0s + Train loss: 0.653061 (2657.1s) ETA: 21917min + Val loss: 0.708313 [t_0.0-0.2=0.9434 t_0.2-0.4=0.7637 t_0.4-0.6=0.6467 t_0.6-0.8=0.5652 t_0.8-1.0=0.5784] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 23.2/188.5 GiB (12.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 58/499 (12% done) --- + [Epoch 58][10/200] loss=0.657325 avg=0.663826 VRAM=38.6GiB | 11.6% done | ETA(epoch): 2518s + [Epoch 58][20/200] loss=0.793082 avg=0.680155 VRAM=38.5GiB | 11.6% done | ETA(epoch): 2388s + [Epoch 58][30/200] loss=0.696354 avg=0.680110 VRAM=38.6GiB | 11.6% done | ETA(epoch): 2255s + [Epoch 58][40/200] loss=0.653807 avg=0.680795 VRAM=38.5GiB | 11.6% done | ETA(epoch): 2122s + [Epoch 58][50/200] loss=0.622447 avg=0.668159 VRAM=38.6GiB | 11.7% done | ETA(epoch): 1989s + [Epoch 58][60/200] loss=0.738841 avg=0.666130 VRAM=38.5GiB | 11.7% done | ETA(epoch): 1858s + [Epoch 58][70/200] loss=0.654675 avg=0.665718 VRAM=38.6GiB | 11.7% done | ETA(epoch): 1725s + [Epoch 58][80/200] loss=0.660008 avg=0.665277 VRAM=38.5GiB | 11.7% done | ETA(epoch): 1591s + [Epoch 58][90/200] loss=0.507982 avg=0.659832 VRAM=38.6GiB | 11.7% done | ETA(epoch): 1459s + [Epoch 58][100/200] loss=0.641181 avg=0.657309 VRAM=38.5GiB | 11.7% done | ETA(epoch): 1326s + [Epoch 58][110/200] loss=0.554137 avg=0.657616 VRAM=38.6GiB | 11.7% done | ETA(epoch): 1194s + [Epoch 58][120/200] loss=0.666043 avg=0.658849 VRAM=38.5GiB | 11.7% done | ETA(epoch): 1063s + [Epoch 58][130/200] loss=0.708927 avg=0.659517 VRAM=38.6GiB | 11.7% done | ETA(epoch): 930s + [Epoch 58][140/200] loss=0.662692 avg=0.659380 VRAM=38.5GiB | 11.7% done | ETA(epoch): 797s + [Epoch 58][150/200] loss=0.670418 avg=0.659178 VRAM=38.6GiB | 11.8% done | ETA(epoch): 664s + [Epoch 58][160/200] loss=0.654221 avg=0.655983 VRAM=38.5GiB | 11.8% done | ETA(epoch): 531s + [Epoch 58][170/200] loss=0.770877 avg=0.656891 VRAM=38.6GiB | 11.8% done | ETA(epoch): 398s + [Epoch 58][180/200] loss=0.633136 avg=0.656520 VRAM=38.5GiB | 11.8% done | ETA(epoch): 266s + [Epoch 58][190/200] loss=0.681560 avg=0.656548 VRAM=38.6GiB | 11.8% done | ETA(epoch): 133s + [Epoch 58][200/200] loss=0.715129 avg=0.657600 VRAM=38.5GiB | 11.8% done | ETA(epoch): 0s + Train loss: 0.657600 (2655.3s) ETA: 21859min + Val loss: 0.709622 [t_0.0-0.2=0.9481 t_0.2-0.4=0.7590 t_0.4-0.6=0.6492 t_0.6-0.8=0.5704 t_0.8-1.0=0.5732] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 23.4/188.5 GiB (12.4%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 59/499 (12% done) --- + [Epoch 59][10/200] loss=0.786040 avg=0.628222 VRAM=38.6GiB | 11.8% done | ETA(epoch): 2518s + [Epoch 59][20/200] loss=0.584628 avg=0.649421 VRAM=38.5GiB | 11.8% done | ETA(epoch): 2386s + [Epoch 59][30/200] loss=0.671705 avg=0.646987 VRAM=38.6GiB | 11.8% done | ETA(epoch): 2255s + [Epoch 59][40/200] loss=0.666720 avg=0.648369 VRAM=38.5GiB | 11.8% done | ETA(epoch): 2122s + [Epoch 59][50/200] loss=0.726431 avg=0.653278 VRAM=38.6GiB | 11.8% done | ETA(epoch): 1990s + [Epoch 59][60/200] loss=0.672020 avg=0.654098 VRAM=38.5GiB | 11.9% done | ETA(epoch): 1857s + [Epoch 59][70/200] loss=0.612369 avg=0.654581 VRAM=38.6GiB | 11.9% done | ETA(epoch): 1725s + [Epoch 59][80/200] loss=0.535095 avg=0.660214 VRAM=38.5GiB | 11.9% done | ETA(epoch): 1598s + [Epoch 59][90/200] loss=0.571026 avg=0.655393 VRAM=38.6GiB | 11.9% done | ETA(epoch): 1464s + [Epoch 59][100/200] loss=0.713877 avg=0.658728 VRAM=38.5GiB | 11.9% done | ETA(epoch): 1330s + [Epoch 59][110/200] loss=0.654778 avg=0.657740 VRAM=38.6GiB | 11.9% done | ETA(epoch): 1197s + [Epoch 59][120/200] loss=0.664375 avg=0.659208 VRAM=38.5GiB | 11.9% done | ETA(epoch): 1064s + [Epoch 59][130/200] loss=0.804142 avg=0.659593 VRAM=38.6GiB | 11.9% done | ETA(epoch): 931s + [Epoch 59][140/200] loss=0.470554 avg=0.659336 VRAM=38.5GiB | 11.9% done | ETA(epoch): 798s + [Epoch 59][150/200] loss=0.626757 avg=0.660256 VRAM=38.6GiB | 11.9% done | ETA(epoch): 665s + [Epoch 59][160/200] loss=0.641693 avg=0.660833 VRAM=38.5GiB | 12.0% done | ETA(epoch): 532s + [Epoch 59][170/200] loss=0.632042 avg=0.661324 VRAM=38.6GiB | 12.0% done | ETA(epoch): 399s + [Epoch 59][180/200] loss=0.649375 avg=0.660642 VRAM=38.5GiB | 12.0% done | ETA(epoch): 266s + [Epoch 59][190/200] loss=0.644009 avg=0.660489 VRAM=38.6GiB | 12.0% done | ETA(epoch): 133s + [Epoch 59][200/200] loss=0.692784 avg=0.663795 VRAM=38.5GiB | 12.0% done | ETA(epoch): 0s + Train loss: 0.663795 (2660.5s) ETA: 21802min + Val loss: 0.703057 [t_0.0-0.2=0.9491 t_0.2-0.4=0.7620 t_0.4-0.6=0.6316 t_0.6-0.8=0.5655 t_0.8-1.0=0.5965] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 23.3/188.5 GiB (12.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 60/499 (12% done) --- + [Epoch 60][10/200] loss=0.678873 avg=0.685732 VRAM=38.6GiB | 12.0% done | ETA(epoch): 2522s + [Epoch 60][20/200] loss=0.692514 avg=0.667417 VRAM=38.5GiB | 12.0% done | ETA(epoch): 2387s + [Epoch 60][30/200] loss=0.880438 avg=0.663023 VRAM=38.6GiB | 12.0% done | ETA(epoch): 2255s + [Epoch 60][40/200] loss=0.647547 avg=0.666882 VRAM=38.5GiB | 12.0% done | ETA(epoch): 2133s + [Epoch 60][50/200] loss=0.667505 avg=0.657429 VRAM=38.6GiB | 12.0% done | ETA(epoch): 1998s + [Epoch 60][60/200] loss=0.684169 avg=0.658642 VRAM=38.5GiB | 12.1% done | ETA(epoch): 1863s + [Epoch 60][70/200] loss=0.527809 avg=0.655913 VRAM=38.6GiB | 12.1% done | ETA(epoch): 1729s + [Epoch 60][80/200] loss=0.759422 avg=0.657699 VRAM=38.5GiB | 12.1% done | ETA(epoch): 1596s + [Epoch 60][90/200] loss=0.723154 avg=0.658188 VRAM=38.6GiB | 12.1% done | ETA(epoch): 1462s + [Epoch 60][100/200] loss=0.822437 avg=0.662260 VRAM=38.5GiB | 12.1% done | ETA(epoch): 1329s + [Epoch 60][110/200] loss=0.626001 avg=0.661588 VRAM=38.6GiB | 12.1% done | ETA(epoch): 1196s + [Epoch 60][120/200] loss=0.620839 avg=0.661535 VRAM=38.5GiB | 12.1% done | ETA(epoch): 1063s + [Epoch 60][130/200] loss=0.519389 avg=0.662569 VRAM=38.6GiB | 12.1% done | ETA(epoch): 930s + [Epoch 60][140/200] loss=0.585150 avg=0.661174 VRAM=38.5GiB | 12.1% done | ETA(epoch): 799s + [Epoch 60][150/200] loss=0.680713 avg=0.663017 VRAM=38.6GiB | 12.2% done | ETA(epoch): 666s + [Epoch 60][160/200] loss=0.582140 avg=0.660497 VRAM=38.5GiB | 12.2% done | ETA(epoch): 533s + [Epoch 60][170/200] loss=0.776568 avg=0.659850 VRAM=38.6GiB | 12.2% done | ETA(epoch): 399s + [Epoch 60][180/200] loss=0.598636 avg=0.660528 VRAM=38.5GiB | 12.2% done | ETA(epoch): 266s + [Epoch 60][190/200] loss=0.671370 avg=0.661143 VRAM=38.6GiB | 12.2% done | ETA(epoch): 133s + [Epoch 60][200/200] loss=0.590748 avg=0.660892 VRAM=38.5GiB | 12.2% done | ETA(epoch): 0s + Train loss: 0.660892 (2660.9s) ETA: 21745min + Val loss: 0.716155 [t_0.0-0.2=0.9270 t_0.2-0.4=0.7487 t_0.4-0.6=0.6516 t_0.6-0.8=0.5665 t_0.8-1.0=0.5887] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 61/499 (12% done) --- + [Epoch 61][10/200] loss=0.719894 avg=0.683924 VRAM=38.6GiB | 12.2% done | ETA(epoch): 2516s + [Epoch 61][20/200] loss=0.700362 avg=0.663826 VRAM=38.5GiB | 12.2% done | ETA(epoch): 2385s + [Epoch 61][30/200] loss=0.596317 avg=0.658683 VRAM=38.6GiB | 12.2% done | ETA(epoch): 2254s + [Epoch 61][40/200] loss=0.598146 avg=0.658291 VRAM=38.5GiB | 12.2% done | ETA(epoch): 2122s + [Epoch 61][50/200] loss=0.604880 avg=0.663548 VRAM=38.6GiB | 12.2% done | ETA(epoch): 1989s + [Epoch 61][60/200] loss=0.525366 avg=0.659094 VRAM=38.5GiB | 12.3% done | ETA(epoch): 1856s + [Epoch 61][70/200] loss=0.632399 avg=0.661928 VRAM=38.6GiB | 12.3% done | ETA(epoch): 1723s + [Epoch 61][80/200] loss=0.660380 avg=0.661352 VRAM=38.5GiB | 12.3% done | ETA(epoch): 1591s + [Epoch 61][90/200] loss=0.687294 avg=0.662794 VRAM=38.6GiB | 12.3% done | ETA(epoch): 1458s + [Epoch 61][100/200] loss=0.616513 avg=0.665460 VRAM=38.5GiB | 12.3% done | ETA(epoch): 1328s + [Epoch 61][110/200] loss=0.717823 avg=0.667982 VRAM=38.6GiB | 12.3% done | ETA(epoch): 1195s + [Epoch 61][120/200] loss=0.674212 avg=0.668116 VRAM=38.5GiB | 12.3% done | ETA(epoch): 1062s + [Epoch 61][130/200] loss=0.592804 avg=0.665240 VRAM=38.6GiB | 12.3% done | ETA(epoch): 929s + [Epoch 61][140/200] loss=0.669758 avg=0.668158 VRAM=38.5GiB | 12.3% done | ETA(epoch): 797s + [Epoch 61][150/200] loss=0.585750 avg=0.669336 VRAM=38.6GiB | 12.3% done | ETA(epoch): 664s + [Epoch 61][160/200] loss=0.684623 avg=0.670209 VRAM=38.5GiB | 12.4% done | ETA(epoch): 531s + [Epoch 61][170/200] loss=0.622306 avg=0.666895 VRAM=38.6GiB | 12.4% done | ETA(epoch): 398s + [Epoch 61][180/200] loss=0.735291 avg=0.667205 VRAM=38.5GiB | 12.4% done | ETA(epoch): 266s + [Epoch 61][190/200] loss=0.682546 avg=0.668388 VRAM=38.6GiB | 12.4% done | ETA(epoch): 133s + [Epoch 61][200/200] loss=0.615653 avg=0.668959 VRAM=38.5GiB | 12.4% done | ETA(epoch): 0s + Train loss: 0.668959 (2655.2s) ETA: 21687min + Val loss: 0.683917 [t_0.0-0.2=0.9339 t_0.2-0.4=0.7570 t_0.4-0.6=0.6510 t_0.6-0.8=0.5492 t_0.8-1.0=0.5802] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 62/499 (12% done) --- + [Epoch 62][10/200] loss=0.705508 avg=0.677955 VRAM=38.6GiB | 12.4% done | ETA(epoch): 2523s + [Epoch 62][20/200] loss=0.551008 avg=0.674233 VRAM=38.5GiB | 12.4% done | ETA(epoch): 2387s + [Epoch 62][30/200] loss=0.572806 avg=0.670281 VRAM=38.6GiB | 12.4% done | ETA(epoch): 2255s + [Epoch 62][40/200] loss=0.765639 avg=0.674370 VRAM=38.5GiB | 12.4% done | ETA(epoch): 2122s + [Epoch 62][50/200] loss=0.566480 avg=0.665726 VRAM=38.6GiB | 12.4% done | ETA(epoch): 1989s + [Epoch 62][60/200] loss=0.724488 avg=0.665267 VRAM=38.5GiB | 12.5% done | ETA(epoch): 1863s + [Epoch 62][70/200] loss=0.684396 avg=0.662751 VRAM=38.6GiB | 12.5% done | ETA(epoch): 1730s + [Epoch 62][80/200] loss=0.659142 avg=0.663601 VRAM=38.5GiB | 12.5% done | ETA(epoch): 1596s + [Epoch 62][90/200] loss=0.608145 avg=0.662453 VRAM=38.6GiB | 12.5% done | ETA(epoch): 1463s + [Epoch 62][100/200] loss=0.667542 avg=0.661377 VRAM=38.5GiB | 12.5% done | ETA(epoch): 1329s + [Epoch 62][110/200] loss=0.903829 avg=0.659911 VRAM=38.6GiB | 12.5% done | ETA(epoch): 1196s + [Epoch 62][120/200] loss=0.631210 avg=0.659483 VRAM=38.5GiB | 12.5% done | ETA(epoch): 1063s + [Epoch 62][130/200] loss=0.564327 avg=0.656902 VRAM=38.6GiB | 12.5% done | ETA(epoch): 930s + [Epoch 62][140/200] loss=0.736021 avg=0.655021 VRAM=38.5GiB | 12.5% done | ETA(epoch): 797s + [Epoch 62][150/200] loss=0.734988 avg=0.656584 VRAM=38.6GiB | 12.6% done | ETA(epoch): 665s + [Epoch 62][160/200] loss=0.795168 avg=0.656412 VRAM=38.5GiB | 12.6% done | ETA(epoch): 532s + [Epoch 62][170/200] loss=0.737032 avg=0.657846 VRAM=38.6GiB | 12.6% done | ETA(epoch): 399s + [Epoch 62][180/200] loss=0.703171 avg=0.657977 VRAM=38.5GiB | 12.6% done | ETA(epoch): 266s + [Epoch 62][190/200] loss=0.627567 avg=0.659755 VRAM=38.6GiB | 12.6% done | ETA(epoch): 133s + [Epoch 62][200/200] loss=0.642658 avg=0.660002 VRAM=38.5GiB | 12.6% done | ETA(epoch): 0s + Train loss: 0.660002 (2659.4s) ETA: 21630min + Val loss: 0.705472 [t_0.0-0.2=0.9277 t_0.2-0.4=0.7550 t_0.4-0.6=0.6515 t_0.6-0.8=0.5532 t_0.8-1.0=0.5930] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 63/499 (13% done) --- + [Epoch 63][10/200] loss=0.684929 avg=0.706302 VRAM=38.6GiB | 12.6% done | ETA(epoch): 2521s + [Epoch 63][20/200] loss=0.587226 avg=0.692529 VRAM=38.5GiB | 12.6% done | ETA(epoch): 2414s + [Epoch 63][30/200] loss=0.718971 avg=0.698977 VRAM=38.6GiB | 12.6% done | ETA(epoch): 2271s + [Epoch 63][40/200] loss=0.604232 avg=0.685595 VRAM=38.5GiB | 12.6% done | ETA(epoch): 2135s + [Epoch 63][50/200] loss=0.797740 avg=0.685219 VRAM=38.6GiB | 12.7% done | ETA(epoch): 1999s + [Epoch 63][60/200] loss=0.680394 avg=0.682612 VRAM=38.5GiB | 12.7% done | ETA(epoch): 1864s + [Epoch 63][70/200] loss=0.774943 avg=0.674805 VRAM=38.6GiB | 12.7% done | ETA(epoch): 1730s + [Epoch 63][80/200] loss=0.692177 avg=0.677532 VRAM=38.5GiB | 12.7% done | ETA(epoch): 1596s + [Epoch 63][90/200] loss=0.689494 avg=0.677462 VRAM=38.6GiB | 12.7% done | ETA(epoch): 1463s + [Epoch 63][100/200] loss=0.710957 avg=0.676928 VRAM=38.5GiB | 12.7% done | ETA(epoch): 1330s + [Epoch 63][110/200] loss=0.564789 avg=0.676290 VRAM=38.6GiB | 12.7% done | ETA(epoch): 1201s + [Epoch 63][120/200] loss=0.756821 avg=0.675907 VRAM=38.5GiB | 12.7% done | ETA(epoch): 1067s + [Epoch 63][130/200] loss=0.638261 avg=0.677597 VRAM=38.6GiB | 12.7% done | ETA(epoch): 933s + [Epoch 63][140/200] loss=0.779039 avg=0.675781 VRAM=38.5GiB | 12.7% done | ETA(epoch): 800s + [Epoch 63][150/200] loss=0.674097 avg=0.675232 VRAM=38.6GiB | 12.8% done | ETA(epoch): 666s + [Epoch 63][160/200] loss=0.769279 avg=0.675294 VRAM=38.5GiB | 12.8% done | ETA(epoch): 533s + [Epoch 63][170/200] loss=0.670371 avg=0.676054 VRAM=38.6GiB | 12.8% done | ETA(epoch): 399s + [Epoch 63][180/200] loss=0.612207 avg=0.675210 VRAM=38.5GiB | 12.8% done | ETA(epoch): 266s + [Epoch 63][190/200] loss=0.678592 avg=0.673689 VRAM=38.6GiB | 12.8% done | ETA(epoch): 133s + [Epoch 63][200/200] loss=0.613117 avg=0.672827 VRAM=38.5GiB | 12.8% done | ETA(epoch): 0s + Train loss: 0.672827 (2661.3s) ETA: 21574min + Val loss: 0.688460 [t_0.0-0.2=0.9484 t_0.2-0.4=0.7557 t_0.4-0.6=0.6496 t_0.6-0.8=0.5750 t_0.8-1.0=0.5711] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 64/499 (13% done) --- + [Epoch 64][10/200] loss=0.697019 avg=0.679913 VRAM=38.6GiB | 12.8% done | ETA(epoch): 2519s + [Epoch 64][20/200] loss=0.718390 avg=0.680940 VRAM=38.5GiB | 12.8% done | ETA(epoch): 2390s + [Epoch 64][30/200] loss=0.632442 avg=0.672628 VRAM=38.6GiB | 12.8% done | ETA(epoch): 2255s + [Epoch 64][40/200] loss=0.560516 avg=0.674961 VRAM=38.5GiB | 12.8% done | ETA(epoch): 2123s + [Epoch 64][50/200] loss=0.589758 avg=0.673150 VRAM=38.6GiB | 12.8% done | ETA(epoch): 1990s + [Epoch 64][60/200] loss=0.622634 avg=0.670411 VRAM=38.5GiB | 12.9% done | ETA(epoch): 1857s + [Epoch 64][70/200] loss=0.637398 avg=0.661491 VRAM=38.6GiB | 12.9% done | ETA(epoch): 1724s + [Epoch 64][80/200] loss=0.819987 avg=0.660431 VRAM=38.5GiB | 12.9% done | ETA(epoch): 1592s + [Epoch 64][90/200] loss=0.558479 avg=0.658615 VRAM=38.6GiB | 12.9% done | ETA(epoch): 1459s + [Epoch 64][100/200] loss=0.737201 avg=0.657957 VRAM=38.5GiB | 12.9% done | ETA(epoch): 1327s + [Epoch 64][110/200] loss=0.577432 avg=0.657234 VRAM=38.6GiB | 12.9% done | ETA(epoch): 1194s + [Epoch 64][120/200] loss=0.754817 avg=0.658015 VRAM=38.5GiB | 12.9% done | ETA(epoch): 1061s + [Epoch 64][130/200] loss=0.747037 avg=0.659363 VRAM=38.6GiB | 12.9% done | ETA(epoch): 928s + [Epoch 64][140/200] loss=0.662896 avg=0.660882 VRAM=38.5GiB | 12.9% done | ETA(epoch): 796s + [Epoch 64][150/200] loss=0.818755 avg=0.659877 VRAM=38.6GiB | 13.0% done | ETA(epoch): 663s + [Epoch 64][160/200] loss=0.745667 avg=0.660945 VRAM=38.5GiB | 13.0% done | ETA(epoch): 531s + [Epoch 64][170/200] loss=0.637761 avg=0.659859 VRAM=38.6GiB | 13.0% done | ETA(epoch): 398s + [Epoch 64][180/200] loss=0.659892 avg=0.660936 VRAM=38.5GiB | 13.0% done | ETA(epoch): 265s + [Epoch 64][190/200] loss=0.777399 avg=0.661635 VRAM=38.6GiB | 13.0% done | ETA(epoch): 133s + [Epoch 64][200/200] loss=0.543762 avg=0.661316 VRAM=38.5GiB | 13.0% done | ETA(epoch): 0s + Train loss: 0.661316 (2652.9s) ETA: 21517min + Val loss: 0.687498 [t_0.0-0.2=0.9400 t_0.2-0.4=0.7481 t_0.4-0.6=0.6414 t_0.6-0.8=0.5749 t_0.8-1.0=0.5722] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 65/499 (13% done) --- + [Epoch 65][10/200] loss=0.481836 avg=0.665713 VRAM=38.6GiB | 13.0% done | ETA(epoch): 2517s + [Epoch 65][20/200] loss=0.640364 avg=0.663223 VRAM=38.5GiB | 13.0% done | ETA(epoch): 2387s + [Epoch 65][30/200] loss=0.665018 avg=0.653513 VRAM=38.6GiB | 13.0% done | ETA(epoch): 2254s + [Epoch 65][40/200] loss=0.833372 avg=0.668712 VRAM=38.5GiB | 13.0% done | ETA(epoch): 2124s + [Epoch 65][50/200] loss=0.654188 avg=0.675407 VRAM=38.6GiB | 13.1% done | ETA(epoch): 1990s + [Epoch 65][60/200] loss=0.660474 avg=0.679320 VRAM=38.5GiB | 13.1% done | ETA(epoch): 1857s + [Epoch 65][70/200] loss=0.553461 avg=0.674891 VRAM=38.6GiB | 13.1% done | ETA(epoch): 1725s + [Epoch 65][80/200] loss=0.729583 avg=0.676259 VRAM=38.5GiB | 13.1% done | ETA(epoch): 1592s + [Epoch 65][90/200] loss=0.826099 avg=0.669918 VRAM=38.6GiB | 13.1% done | ETA(epoch): 1459s + [Epoch 65][100/200] loss=0.582592 avg=0.665705 VRAM=38.5GiB | 13.1% done | ETA(epoch): 1327s + [Epoch 65][110/200] loss=0.659305 avg=0.667438 VRAM=38.6GiB | 13.1% done | ETA(epoch): 1194s + [Epoch 65][120/200] loss=0.657907 avg=0.665247 VRAM=38.5GiB | 13.1% done | ETA(epoch): 1061s + [Epoch 65][130/200] loss=0.614625 avg=0.667446 VRAM=38.6GiB | 13.1% done | ETA(epoch): 930s + [Epoch 65][140/200] loss=0.630989 avg=0.664235 VRAM=38.5GiB | 13.1% done | ETA(epoch): 797s + [Epoch 65][150/200] loss=0.768113 avg=0.663307 VRAM=38.6GiB | 13.2% done | ETA(epoch): 664s + [Epoch 65][160/200] loss=0.622652 avg=0.665874 VRAM=38.5GiB | 13.2% done | ETA(epoch): 531s + [Epoch 65][170/200] loss=0.717707 avg=0.664918 VRAM=38.6GiB | 13.2% done | ETA(epoch): 398s + [Epoch 65][180/200] loss=0.655378 avg=0.664953 VRAM=38.5GiB | 13.2% done | ETA(epoch): 266s + [Epoch 65][190/200] loss=0.725819 avg=0.665475 VRAM=38.6GiB | 13.2% done | ETA(epoch): 133s + [Epoch 65][200/200] loss=0.692589 avg=0.664048 VRAM=38.5GiB | 13.2% done | ETA(epoch): 0s + Train loss: 0.664048 (2655.5s) ETA: 21461min + Val loss: 0.699672 [t_0.0-0.2=0.9423 t_0.2-0.4=0.7536 t_0.4-0.6=0.6585 t_0.6-0.8=0.5655 t_0.8-1.0=0.5667] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 66/499 (13% done) --- + [Epoch 66][10/200] loss=0.578641 avg=0.701705 VRAM=38.6GiB | 13.2% done | ETA(epoch): 2521s + [Epoch 66][20/200] loss=0.660757 avg=0.705445 VRAM=38.5GiB | 13.2% done | ETA(epoch): 2387s + [Epoch 66][30/200] loss=0.609494 avg=0.703682 VRAM=38.6GiB | 13.2% done | ETA(epoch): 2256s + [Epoch 66][40/200] loss=0.601682 avg=0.691748 VRAM=38.5GiB | 13.2% done | ETA(epoch): 2123s + [Epoch 66][50/200] loss=0.629440 avg=0.690104 VRAM=38.6GiB | 13.2% done | ETA(epoch): 1990s + [Epoch 66][60/200] loss=0.701622 avg=0.678048 VRAM=38.5GiB | 13.3% done | ETA(epoch): 1857s + [Epoch 66][70/200] loss=0.671516 avg=0.673255 VRAM=38.6GiB | 13.3% done | ETA(epoch): 1725s + [Epoch 66][80/200] loss=0.596910 avg=0.668441 VRAM=38.5GiB | 13.3% done | ETA(epoch): 1592s + [Epoch 66][90/200] loss=0.779065 avg=0.667936 VRAM=38.6GiB | 13.3% done | ETA(epoch): 1459s + [Epoch 66][100/200] loss=0.587380 avg=0.668685 VRAM=38.5GiB | 13.3% done | ETA(epoch): 1327s + [Epoch 66][110/200] loss=0.781590 avg=0.669552 VRAM=38.6GiB | 13.3% done | ETA(epoch): 1194s + [Epoch 66][120/200] loss=0.704592 avg=0.671003 VRAM=38.5GiB | 13.3% done | ETA(epoch): 1061s + [Epoch 66][130/200] loss=0.649858 avg=0.669632 VRAM=38.6GiB | 13.3% done | ETA(epoch): 929s + [Epoch 66][140/200] loss=0.593712 avg=0.666911 VRAM=38.5GiB | 13.3% done | ETA(epoch): 796s + [Epoch 66][150/200] loss=0.660389 avg=0.665553 VRAM=38.6GiB | 13.4% done | ETA(epoch): 663s + [Epoch 66][160/200] loss=0.613303 avg=0.664450 VRAM=38.5GiB | 13.4% done | ETA(epoch): 531s + [Epoch 66][170/200] loss=0.549092 avg=0.665146 VRAM=38.6GiB | 13.4% done | ETA(epoch): 398s + [Epoch 66][180/200] loss=0.640682 avg=0.665050 VRAM=38.5GiB | 13.4% done | ETA(epoch): 265s + [Epoch 66][190/200] loss=0.608053 avg=0.663955 VRAM=38.6GiB | 13.4% done | ETA(epoch): 133s + [Epoch 66][200/200] loss=0.621044 avg=0.662867 VRAM=38.5GiB | 13.4% done | ETA(epoch): 0s + Train loss: 0.662867 (2653.0s) ETA: 21404min + Val loss: 0.689232 [t_0.0-0.2=0.9078 t_0.2-0.4=0.7628 t_0.4-0.6=0.6512 t_0.6-0.8=0.5468 t_0.8-1.0=0.5786] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 67/499 (13% done) --- + [Epoch 67][10/200] loss=0.707831 avg=0.688276 VRAM=38.6GiB | 13.4% done | ETA(epoch): 2515s + [Epoch 67][20/200] loss=0.590751 avg=0.691117 VRAM=38.5GiB | 13.4% done | ETA(epoch): 2385s + [Epoch 67][30/200] loss=0.690164 avg=0.679590 VRAM=38.6GiB | 13.4% done | ETA(epoch): 2252s + [Epoch 67][40/200] loss=0.590417 avg=0.680076 VRAM=38.5GiB | 13.4% done | ETA(epoch): 2120s + [Epoch 67][50/200] loss=0.634869 avg=0.673410 VRAM=38.6GiB | 13.5% done | ETA(epoch): 1988s + [Epoch 67][60/200] loss=0.663185 avg=0.677827 VRAM=38.5GiB | 13.5% done | ETA(epoch): 1855s + [Epoch 67][70/200] loss=0.720180 avg=0.677630 VRAM=38.6GiB | 13.5% done | ETA(epoch): 1723s + [Epoch 67][80/200] loss=0.540251 avg=0.673103 VRAM=38.5GiB | 13.5% done | ETA(epoch): 1590s + [Epoch 67][90/200] loss=0.671128 avg=0.668016 VRAM=38.6GiB | 13.5% done | ETA(epoch): 1458s + [Epoch 67][100/200] loss=0.627974 avg=0.669912 VRAM=38.5GiB | 13.5% done | ETA(epoch): 1326s + [Epoch 67][110/200] loss=0.669492 avg=0.671582 VRAM=38.6GiB | 13.5% done | ETA(epoch): 1193s + [Epoch 67][120/200] loss=0.665358 avg=0.670515 VRAM=38.5GiB | 13.5% done | ETA(epoch): 1060s + [Epoch 67][130/200] loss=0.772002 avg=0.671076 VRAM=38.6GiB | 13.5% done | ETA(epoch): 928s + [Epoch 67][140/200] loss=0.696225 avg=0.671314 VRAM=38.5GiB | 13.5% done | ETA(epoch): 795s + [Epoch 67][150/200] loss=0.677056 avg=0.670120 VRAM=38.6GiB | 13.6% done | ETA(epoch): 663s + [Epoch 67][160/200] loss=0.872833 avg=0.669366 VRAM=38.5GiB | 13.6% done | ETA(epoch): 530s + [Epoch 67][170/200] loss=0.574950 avg=0.668188 VRAM=38.6GiB | 13.6% done | ETA(epoch): 398s + [Epoch 67][180/200] loss=0.578266 avg=0.667769 VRAM=38.5GiB | 13.6% done | ETA(epoch): 265s + [Epoch 67][190/200] loss=0.596390 avg=0.666285 VRAM=38.6GiB | 13.6% done | ETA(epoch): 133s + [Epoch 67][200/200] loss=0.615261 avg=0.666341 VRAM=38.5GiB | 13.6% done | ETA(epoch): 0s + Train loss: 0.666341 (2651.6s) ETA: 21348min + Val loss: 0.691632 [t_0.0-0.2=0.9295 t_0.2-0.4=0.7527 t_0.4-0.6=0.6605 t_0.6-0.8=0.5662 t_0.8-1.0=0.5599] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 68/499 (14% done) --- + [Epoch 68][10/200] loss=0.770729 avg=0.659201 VRAM=38.6GiB | 13.6% done | ETA(epoch): 2518s + [Epoch 68][20/200] loss=0.718070 avg=0.656355 VRAM=38.5GiB | 13.6% done | ETA(epoch): 2386s + [Epoch 68][30/200] loss=0.821042 avg=0.655154 VRAM=38.6GiB | 13.6% done | ETA(epoch): 2253s + [Epoch 68][40/200] loss=0.732397 avg=0.659491 VRAM=38.5GiB | 13.6% done | ETA(epoch): 2122s + [Epoch 68][50/200] loss=0.548176 avg=0.653215 VRAM=38.6GiB | 13.7% done | ETA(epoch): 1989s + [Epoch 68][60/200] loss=0.640425 avg=0.653918 VRAM=38.5GiB | 13.7% done | ETA(epoch): 1857s + [Epoch 68][70/200] loss=0.615650 avg=0.656952 VRAM=38.6GiB | 13.7% done | ETA(epoch): 1724s + [Epoch 68][80/200] loss=0.683613 avg=0.653994 VRAM=38.5GiB | 13.7% done | ETA(epoch): 1592s + [Epoch 68][90/200] loss=0.714614 avg=0.654882 VRAM=38.6GiB | 13.7% done | ETA(epoch): 1459s + [Epoch 68][100/200] loss=0.556877 avg=0.656949 VRAM=38.5GiB | 13.7% done | ETA(epoch): 1326s + [Epoch 68][110/200] loss=0.633189 avg=0.658519 VRAM=38.6GiB | 13.7% done | ETA(epoch): 1193s + [Epoch 68][120/200] loss=0.654699 avg=0.656906 VRAM=38.5GiB | 13.7% done | ETA(epoch): 1061s + [Epoch 68][130/200] loss=0.620749 avg=0.659288 VRAM=38.6GiB | 13.7% done | ETA(epoch): 928s + [Epoch 68][140/200] loss=0.579636 avg=0.658345 VRAM=38.5GiB | 13.7% done | ETA(epoch): 796s + [Epoch 68][150/200] loss=0.748327 avg=0.659765 VRAM=38.6GiB | 13.8% done | ETA(epoch): 663s + [Epoch 68][160/200] loss=0.660060 avg=0.657150 VRAM=38.5GiB | 13.8% done | ETA(epoch): 530s + [Epoch 68][170/200] loss=0.589241 avg=0.657373 VRAM=38.6GiB | 13.8% done | ETA(epoch): 398s + [Epoch 68][180/200] loss=0.662281 avg=0.655699 VRAM=38.5GiB | 13.8% done | ETA(epoch): 265s + [Epoch 68][190/200] loss=0.720518 avg=0.657230 VRAM=38.6GiB | 13.8% done | ETA(epoch): 133s + [Epoch 68][200/200] loss=0.608994 avg=0.656375 VRAM=38.5GiB | 13.8% done | ETA(epoch): 0s + Train loss: 0.656375 (2654.4s) ETA: 21292min + Val loss: 0.696457 [t_0.0-0.2=0.9340 t_0.2-0.4=0.7509 t_0.4-0.6=0.6463 t_0.6-0.8=0.5623 t_0.8-1.0=0.6009] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 69/499 (14% done) --- + [Epoch 69][10/200] loss=0.617368 avg=0.667848 VRAM=38.6GiB | 13.8% done | ETA(epoch): 2518s + [Epoch 69][20/200] loss=0.662802 avg=0.660014 VRAM=38.5GiB | 13.8% done | ETA(epoch): 2386s + [Epoch 69][30/200] loss=0.634368 avg=0.658086 VRAM=38.6GiB | 13.8% done | ETA(epoch): 2253s + [Epoch 69][40/200] loss=0.712721 avg=0.662959 VRAM=38.5GiB | 13.8% done | ETA(epoch): 2121s + [Epoch 69][50/200] loss=0.520845 avg=0.663797 VRAM=38.6GiB | 13.9% done | ETA(epoch): 1988s + [Epoch 69][60/200] loss=0.675833 avg=0.659026 VRAM=38.5GiB | 13.9% done | ETA(epoch): 1856s + [Epoch 69][70/200] loss=0.714472 avg=0.664812 VRAM=38.6GiB | 13.9% done | ETA(epoch): 1723s + [Epoch 69][80/200] loss=0.645721 avg=0.664102 VRAM=38.5GiB | 13.9% done | ETA(epoch): 1591s + [Epoch 69][90/200] loss=0.760749 avg=0.665838 VRAM=38.6GiB | 13.9% done | ETA(epoch): 1458s + [Epoch 69][100/200] loss=0.606856 avg=0.664937 VRAM=38.5GiB | 13.9% done | ETA(epoch): 1326s + [Epoch 69][110/200] loss=0.678713 avg=0.664461 VRAM=38.6GiB | 13.9% done | ETA(epoch): 1193s + [Epoch 69][120/200] loss=0.685288 avg=0.661438 VRAM=38.5GiB | 13.9% done | ETA(epoch): 1060s + [Epoch 69][130/200] loss=0.531180 avg=0.661627 VRAM=38.6GiB | 13.9% done | ETA(epoch): 928s + [Epoch 69][140/200] loss=0.595712 avg=0.663424 VRAM=38.5GiB | 13.9% done | ETA(epoch): 795s + [Epoch 69][150/200] loss=0.768685 avg=0.667767 VRAM=38.6GiB | 14.0% done | ETA(epoch): 663s + [Epoch 69][160/200] loss=0.716662 avg=0.668734 VRAM=38.5GiB | 14.0% done | ETA(epoch): 531s + [Epoch 69][170/200] loss=0.730981 avg=0.667779 VRAM=38.6GiB | 14.0% done | ETA(epoch): 398s + [Epoch 69][180/200] loss=0.653269 avg=0.665680 VRAM=38.5GiB | 14.0% done | ETA(epoch): 265s + [Epoch 69][190/200] loss=0.636684 avg=0.661786 VRAM=38.6GiB | 14.0% done | ETA(epoch): 133s + [Epoch 69][200/200] loss=0.671103 avg=0.663471 VRAM=38.5GiB | 14.0% done | ETA(epoch): 0s + Train loss: 0.663471 (2653.9s) ETA: 21237min + Val loss: 0.700115 [t_0.0-0.2=0.9318 t_0.2-0.4=0.7557 t_0.4-0.6=0.6391 t_0.6-0.8=0.5585 t_0.8-1.0=0.5865] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 70/499 (14% done) --- + [Epoch 70][10/200] loss=0.692294 avg=0.661749 VRAM=38.6GiB | 14.0% done | ETA(epoch): 2522s + [Epoch 70][20/200] loss=0.620749 avg=0.662638 VRAM=38.5GiB | 14.0% done | ETA(epoch): 2386s + [Epoch 70][30/200] loss=0.582388 avg=0.649373 VRAM=38.6GiB | 14.0% done | ETA(epoch): 2256s + [Epoch 70][40/200] loss=0.677552 avg=0.647957 VRAM=38.5GiB | 14.0% done | ETA(epoch): 2143s + [Epoch 70][50/200] loss=0.554815 avg=0.654307 VRAM=38.6GiB | 14.1% done | ETA(epoch): 2006s + [Epoch 70][60/200] loss=0.611729 avg=0.654903 VRAM=38.5GiB | 14.1% done | ETA(epoch): 1869s + [Epoch 70][70/200] loss=0.637248 avg=0.656085 VRAM=38.6GiB | 14.1% done | ETA(epoch): 1734s + [Epoch 70][80/200] loss=0.635650 avg=0.658140 VRAM=38.5GiB | 14.1% done | ETA(epoch): 1600s + [Epoch 70][90/200] loss=0.718033 avg=0.664232 VRAM=38.6GiB | 14.1% done | ETA(epoch): 1465s + [Epoch 70][100/200] loss=0.767555 avg=0.665436 VRAM=38.5GiB | 14.1% done | ETA(epoch): 1332s + [Epoch 70][110/200] loss=0.623602 avg=0.665393 VRAM=38.6GiB | 14.1% done | ETA(epoch): 1198s + [Epoch 70][120/200] loss=0.638221 avg=0.665175 VRAM=38.5GiB | 14.1% done | ETA(epoch): 1065s + [Epoch 70][130/200] loss=0.654262 avg=0.666732 VRAM=38.6GiB | 14.1% done | ETA(epoch): 934s + [Epoch 70][140/200] loss=0.616146 avg=0.667434 VRAM=38.5GiB | 14.1% done | ETA(epoch): 800s + [Epoch 70][150/200] loss=0.585830 avg=0.665829 VRAM=38.6GiB | 14.1% done | ETA(epoch): 666s + [Epoch 70][160/200] loss=0.558663 avg=0.665096 VRAM=38.5GiB | 14.2% done | ETA(epoch): 533s + [Epoch 70][170/200] loss=0.634325 avg=0.663866 VRAM=38.6GiB | 14.2% done | ETA(epoch): 400s + [Epoch 70][180/200] loss=0.539909 avg=0.665828 VRAM=38.5GiB | 14.2% done | ETA(epoch): 266s + [Epoch 70][190/200] loss=0.624747 avg=0.664292 VRAM=38.6GiB | 14.2% done | ETA(epoch): 133s + [Epoch 70][200/200] loss=0.694296 avg=0.661708 VRAM=38.5GiB | 14.2% done | ETA(epoch): 0s + Train loss: 0.661708 (2662.8s) ETA: 21182min + Val loss: 0.686920 [t_0.0-0.2=0.9201 t_0.2-0.4=0.7422 t_0.4-0.6=0.6470 t_0.6-0.8=0.5579 t_0.8-1.0=0.5857] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 71/499 (14% done) --- + [Epoch 71][10/200] loss=0.651608 avg=0.663802 VRAM=38.6GiB | 14.2% done | ETA(epoch): 2521s + [Epoch 71][20/200] loss=0.554420 avg=0.650614 VRAM=38.5GiB | 14.2% done | ETA(epoch): 2387s + [Epoch 71][30/200] loss=0.594566 avg=0.649397 VRAM=38.6GiB | 14.2% done | ETA(epoch): 2255s + [Epoch 71][40/200] loss=0.595286 avg=0.649385 VRAM=38.5GiB | 14.2% done | ETA(epoch): 2122s + [Epoch 71][50/200] loss=0.687879 avg=0.658913 VRAM=38.6GiB | 14.2% done | ETA(epoch): 1989s + [Epoch 71][60/200] loss=0.615693 avg=0.655234 VRAM=38.5GiB | 14.3% done | ETA(epoch): 1856s + [Epoch 71][70/200] loss=0.617829 avg=0.654697 VRAM=38.6GiB | 14.3% done | ETA(epoch): 1724s + [Epoch 71][80/200] loss=0.674260 avg=0.655608 VRAM=38.5GiB | 14.3% done | ETA(epoch): 1591s + [Epoch 71][90/200] loss=0.636698 avg=0.655028 VRAM=38.6GiB | 14.3% done | ETA(epoch): 1462s + [Epoch 71][100/200] loss=0.608540 avg=0.655797 VRAM=38.5GiB | 14.3% done | ETA(epoch): 1329s + [Epoch 71][110/200] loss=0.810465 avg=0.659443 VRAM=38.6GiB | 14.3% done | ETA(epoch): 1196s + [Epoch 71][120/200] loss=0.676979 avg=0.660444 VRAM=38.5GiB | 14.3% done | ETA(epoch): 1063s + [Epoch 71][130/200] loss=0.629679 avg=0.659346 VRAM=38.6GiB | 14.3% done | ETA(epoch): 930s + [Epoch 71][140/200] loss=0.649612 avg=0.658081 VRAM=38.5GiB | 14.3% done | ETA(epoch): 797s + [Epoch 71][150/200] loss=0.651593 avg=0.658997 VRAM=38.6GiB | 14.3% done | ETA(epoch): 664s + [Epoch 71][160/200] loss=0.563240 avg=0.657826 VRAM=38.5GiB | 14.4% done | ETA(epoch): 531s + [Epoch 71][170/200] loss=0.743837 avg=0.660290 VRAM=38.6GiB | 14.4% done | ETA(epoch): 398s + [Epoch 71][180/200] loss=0.737811 avg=0.661074 VRAM=38.5GiB | 14.4% done | ETA(epoch): 266s + [Epoch 71][190/200] loss=0.694953 avg=0.661933 VRAM=38.6GiB | 14.4% done | ETA(epoch): 133s + [Epoch 71][200/200] loss=0.740933 avg=0.663438 VRAM=38.5GiB | 14.4% done | ETA(epoch): 0s + Train loss: 0.663438 (2660.7s) ETA: 21127min + Val loss: 0.709790 [t_0.0-0.2=0.9375 t_0.2-0.4=0.7578 t_0.4-0.6=0.6192 t_0.6-0.8=0.5890 t_0.8-1.0=0.5711] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 72/499 (14% done) --- + [Epoch 72][10/200] loss=0.670689 avg=0.651933 VRAM=38.6GiB | 14.4% done | ETA(epoch): 2523s + [Epoch 72][20/200] loss=0.632407 avg=0.646021 VRAM=38.5GiB | 14.4% done | ETA(epoch): 2388s + [Epoch 72][30/200] loss=0.681764 avg=0.650714 VRAM=38.6GiB | 14.4% done | ETA(epoch): 2255s + [Epoch 72][40/200] loss=0.726774 avg=0.650959 VRAM=38.5GiB | 14.4% done | ETA(epoch): 2123s + [Epoch 72][50/200] loss=0.661216 avg=0.648204 VRAM=38.6GiB | 14.4% done | ETA(epoch): 1989s + [Epoch 72][60/200] loss=0.660934 avg=0.648135 VRAM=38.5GiB | 14.5% done | ETA(epoch): 1857s + [Epoch 72][70/200] loss=0.650961 avg=0.649034 VRAM=38.6GiB | 14.5% done | ETA(epoch): 1724s + [Epoch 72][80/200] loss=0.580103 avg=0.649892 VRAM=38.5GiB | 14.5% done | ETA(epoch): 1592s + [Epoch 72][90/200] loss=0.567316 avg=0.649944 VRAM=38.6GiB | 14.5% done | ETA(epoch): 1459s + [Epoch 72][100/200] loss=0.706874 avg=0.653200 VRAM=38.5GiB | 14.5% done | ETA(epoch): 1326s + [Epoch 72][110/200] loss=0.697899 avg=0.655521 VRAM=38.6GiB | 14.5% done | ETA(epoch): 1194s + [Epoch 72][120/200] loss=0.685130 avg=0.655294 VRAM=38.5GiB | 14.5% done | ETA(epoch): 1061s + [Epoch 72][130/200] loss=0.801961 avg=0.657637 VRAM=38.6GiB | 14.5% done | ETA(epoch): 928s + [Epoch 72][140/200] loss=0.602582 avg=0.657302 VRAM=38.5GiB | 14.5% done | ETA(epoch): 797s + [Epoch 72][150/200] loss=0.669201 avg=0.660611 VRAM=38.6GiB | 14.5% done | ETA(epoch): 664s + [Epoch 72][160/200] loss=0.597864 avg=0.659923 VRAM=38.5GiB | 14.6% done | ETA(epoch): 531s + [Epoch 72][170/200] loss=0.610104 avg=0.660611 VRAM=38.6GiB | 14.6% done | ETA(epoch): 398s + [Epoch 72][180/200] loss=0.662038 avg=0.661997 VRAM=38.5GiB | 14.6% done | ETA(epoch): 266s + [Epoch 72][190/200] loss=0.629958 avg=0.659539 VRAM=38.6GiB | 14.6% done | ETA(epoch): 133s + [Epoch 72][200/200] loss=0.600072 avg=0.659878 VRAM=38.5GiB | 14.6% done | ETA(epoch): 0s + Train loss: 0.659878 (2654.7s) ETA: 21072min + Val loss: 0.692494 [t_0.0-0.2=0.9332 t_0.2-0.4=0.7583 t_0.4-0.6=0.6442 t_0.6-0.8=0.5712 t_0.8-1.0=0.5655] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 73/499 (15% done) --- + [Epoch 73][10/200] loss=0.572229 avg=0.628681 VRAM=38.6GiB | 14.6% done | ETA(epoch): 2515s + [Epoch 73][20/200] loss=0.716837 avg=0.677795 VRAM=38.5GiB | 14.6% done | ETA(epoch): 2385s + [Epoch 73][30/200] loss=0.506556 avg=0.676892 VRAM=38.6GiB | 14.6% done | ETA(epoch): 2253s + [Epoch 73][40/200] loss=0.642371 avg=0.680728 VRAM=38.5GiB | 14.6% done | ETA(epoch): 2121s + [Epoch 73][50/200] loss=0.760847 avg=0.687384 VRAM=38.6GiB | 14.6% done | ETA(epoch): 1988s + [Epoch 73][60/200] loss=0.735908 avg=0.682953 VRAM=38.5GiB | 14.7% done | ETA(epoch): 1856s + [Epoch 73][70/200] loss=0.749760 avg=0.685999 VRAM=38.6GiB | 14.7% done | ETA(epoch): 1724s + [Epoch 73][80/200] loss=0.717028 avg=0.682554 VRAM=38.5GiB | 14.7% done | ETA(epoch): 1591s + [Epoch 73][90/200] loss=0.650337 avg=0.677456 VRAM=38.6GiB | 14.7% done | ETA(epoch): 1459s + [Epoch 73][100/200] loss=0.670715 avg=0.674248 VRAM=38.5GiB | 14.7% done | ETA(epoch): 1326s + [Epoch 73][110/200] loss=0.673977 avg=0.674321 VRAM=38.6GiB | 14.7% done | ETA(epoch): 1196s + [Epoch 73][120/200] loss=0.635147 avg=0.676930 VRAM=38.5GiB | 14.7% done | ETA(epoch): 1063s + [Epoch 73][130/200] loss=0.794024 avg=0.677017 VRAM=38.6GiB | 14.7% done | ETA(epoch): 930s + [Epoch 73][140/200] loss=0.684711 avg=0.673204 VRAM=38.5GiB | 14.7% done | ETA(epoch): 797s + [Epoch 73][150/200] loss=0.645020 avg=0.671869 VRAM=38.6GiB | 14.8% done | ETA(epoch): 664s + [Epoch 73][160/200] loss=0.600443 avg=0.671702 VRAM=38.5GiB | 14.8% done | ETA(epoch): 531s + [Epoch 73][170/200] loss=0.659898 avg=0.671894 VRAM=38.6GiB | 14.8% done | ETA(epoch): 398s + [Epoch 73][180/200] loss=0.749904 avg=0.673052 VRAM=38.5GiB | 14.8% done | ETA(epoch): 266s + [Epoch 73][190/200] loss=0.601188 avg=0.671063 VRAM=38.6GiB | 14.8% done | ETA(epoch): 133s + [Epoch 73][200/200] loss=0.857163 avg=0.670405 VRAM=38.5GiB | 14.8% done | ETA(epoch): 0s + Train loss: 0.670405 (2655.6s) ETA: 21017min + Val loss: 0.697200 [t_0.0-0.2=0.9251 t_0.2-0.4=0.7662 t_0.4-0.6=0.6449 t_0.6-0.8=0.5475 t_0.8-1.0=0.5788] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 74/499 (15% done) --- + [Epoch 74][10/200] loss=0.582067 avg=0.671758 VRAM=38.6GiB | 14.8% done | ETA(epoch): 2518s + [Epoch 74][20/200] loss=0.712726 avg=0.680455 VRAM=38.5GiB | 14.8% done | ETA(epoch): 2385s + [Epoch 74][30/200] loss=0.691878 avg=0.673333 VRAM=38.6GiB | 14.8% done | ETA(epoch): 2255s + [Epoch 74][40/200] loss=0.637056 avg=0.659350 VRAM=38.5GiB | 14.8% done | ETA(epoch): 2122s + [Epoch 74][50/200] loss=0.709327 avg=0.661596 VRAM=38.6GiB | 14.8% done | ETA(epoch): 1989s + [Epoch 74][60/200] loss=0.645525 avg=0.663221 VRAM=38.5GiB | 14.9% done | ETA(epoch): 1856s + [Epoch 74][70/200] loss=0.612831 avg=0.664345 VRAM=38.6GiB | 14.9% done | ETA(epoch): 1730s + [Epoch 74][80/200] loss=0.652895 avg=0.663292 VRAM=38.5GiB | 14.9% done | ETA(epoch): 1597s + [Epoch 74][90/200] loss=0.587431 avg=0.661363 VRAM=38.6GiB | 14.9% done | ETA(epoch): 1463s + [Epoch 74][100/200] loss=0.727821 avg=0.662755 VRAM=38.5GiB | 14.9% done | ETA(epoch): 1329s + [Epoch 74][110/200] loss=0.573333 avg=0.661492 VRAM=38.6GiB | 14.9% done | ETA(epoch): 1196s + [Epoch 74][120/200] loss=0.676503 avg=0.662660 VRAM=38.5GiB | 14.9% done | ETA(epoch): 1063s + [Epoch 74][130/200] loss=0.609217 avg=0.664959 VRAM=38.6GiB | 14.9% done | ETA(epoch): 930s + [Epoch 74][140/200] loss=0.746296 avg=0.665557 VRAM=38.5GiB | 14.9% done | ETA(epoch): 797s + [Epoch 74][150/200] loss=0.567802 avg=0.665124 VRAM=38.6GiB | 14.9% done | ETA(epoch): 664s + [Epoch 74][160/200] loss=0.742406 avg=0.664942 VRAM=38.5GiB | 15.0% done | ETA(epoch): 533s + [Epoch 74][170/200] loss=0.694248 avg=0.664147 VRAM=38.6GiB | 15.0% done | ETA(epoch): 399s + [Epoch 74][180/200] loss=0.673915 avg=0.665208 VRAM=38.5GiB | 15.0% done | ETA(epoch): 266s + [Epoch 74][190/200] loss=0.793972 avg=0.666210 VRAM=38.6GiB | 15.0% done | ETA(epoch): 133s + [Epoch 74][200/200] loss=0.689331 avg=0.665665 VRAM=38.5GiB | 15.0% done | ETA(epoch): 0s + Train loss: 0.665665 (2661.1s) ETA: 20963min + Val loss: 0.724308 [t_0.0-0.2=0.9467 t_0.2-0.4=0.7725 t_0.4-0.6=0.6555 t_0.6-0.8=0.5590 t_0.8-1.0=0.5787] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 75/499 (15% done) --- + [Epoch 75][10/200] loss=0.742925 avg=0.655744 VRAM=38.6GiB | 15.0% done | ETA(epoch): 2523s + [Epoch 75][20/200] loss=0.617432 avg=0.667636 VRAM=38.5GiB | 15.0% done | ETA(epoch): 2389s + [Epoch 75][30/200] loss=0.609672 avg=0.658484 VRAM=38.6GiB | 15.0% done | ETA(epoch): 2257s + [Epoch 75][40/200] loss=0.650608 avg=0.647865 VRAM=38.5GiB | 15.0% done | ETA(epoch): 2123s + [Epoch 75][50/200] loss=0.684892 avg=0.653409 VRAM=38.6GiB | 15.0% done | ETA(epoch): 1991s + [Epoch 75][60/200] loss=0.673664 avg=0.650135 VRAM=38.5GiB | 15.1% done | ETA(epoch): 1858s + [Epoch 75][70/200] loss=0.691400 avg=0.655361 VRAM=38.6GiB | 15.1% done | ETA(epoch): 1725s + [Epoch 75][80/200] loss=0.715855 avg=0.654641 VRAM=38.5GiB | 15.1% done | ETA(epoch): 1592s + [Epoch 75][90/200] loss=0.654001 avg=0.651180 VRAM=38.6GiB | 15.1% done | ETA(epoch): 1459s + [Epoch 75][100/200] loss=0.622657 avg=0.660925 VRAM=38.5GiB | 15.1% done | ETA(epoch): 1327s + [Epoch 75][110/200] loss=0.583970 avg=0.658248 VRAM=38.6GiB | 15.1% done | ETA(epoch): 1194s + [Epoch 75][120/200] loss=0.604469 avg=0.658973 VRAM=38.5GiB | 15.1% done | ETA(epoch): 1062s + [Epoch 75][130/200] loss=0.575599 avg=0.658223 VRAM=38.6GiB | 15.1% done | ETA(epoch): 929s + [Epoch 75][140/200] loss=0.705327 avg=0.659656 VRAM=38.5GiB | 15.1% done | ETA(epoch): 796s + [Epoch 75][150/200] loss=0.595802 avg=0.657520 VRAM=38.6GiB | 15.2% done | ETA(epoch): 663s + [Epoch 75][160/200] loss=0.571030 avg=0.657995 VRAM=38.5GiB | 15.2% done | ETA(epoch): 531s + [Epoch 75][170/200] loss=0.697684 avg=0.657188 VRAM=38.6GiB | 15.2% done | ETA(epoch): 398s + [Epoch 75][180/200] loss=0.715074 avg=0.659388 VRAM=38.5GiB | 15.2% done | ETA(epoch): 265s + [Epoch 75][190/200] loss=0.621392 avg=0.658431 VRAM=38.6GiB | 15.2% done | ETA(epoch): 133s + [Epoch 75][200/200] loss=0.622184 avg=0.659962 VRAM=38.5GiB | 15.2% done | ETA(epoch): 0s + Train loss: 0.659962 (2653.3s) ETA: 20909min + Val loss: 0.685691 [t_0.0-0.2=0.9315 t_0.2-0.4=0.7679 t_0.4-0.6=0.6438 t_0.6-0.8=0.5534 t_0.8-1.0=0.5899] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 76/499 (15% done) --- + [Epoch 76][10/200] loss=0.662002 avg=0.700923 VRAM=38.6GiB | 15.2% done | ETA(epoch): 2518s + [Epoch 76][20/200] loss=0.593551 avg=0.668234 VRAM=38.5GiB | 15.2% done | ETA(epoch): 2385s + [Epoch 76][30/200] loss=0.653911 avg=0.664334 VRAM=38.6GiB | 15.2% done | ETA(epoch): 2255s + [Epoch 76][40/200] loss=0.611266 avg=0.668377 VRAM=38.5GiB | 15.2% done | ETA(epoch): 2122s + [Epoch 76][50/200] loss=0.771486 avg=0.670242 VRAM=38.6GiB | 15.2% done | ETA(epoch): 1989s + [Epoch 76][60/200] loss=0.548341 avg=0.663467 VRAM=38.5GiB | 15.3% done | ETA(epoch): 1857s + [Epoch 76][70/200] loss=0.554139 avg=0.664923 VRAM=38.6GiB | 15.3% done | ETA(epoch): 1724s + [Epoch 76][80/200] loss=0.719791 avg=0.666518 VRAM=38.5GiB | 15.3% done | ETA(epoch): 1591s + [Epoch 76][90/200] loss=0.534339 avg=0.667078 VRAM=38.6GiB | 15.3% done | ETA(epoch): 1458s + [Epoch 76][100/200] loss=0.735948 avg=0.670192 VRAM=38.5GiB | 15.3% done | ETA(epoch): 1326s + [Epoch 76][110/200] loss=0.734574 avg=0.668453 VRAM=38.6GiB | 15.3% done | ETA(epoch): 1193s + [Epoch 76][120/200] loss=0.621333 avg=0.670612 VRAM=38.5GiB | 15.3% done | ETA(epoch): 1061s + [Epoch 76][130/200] loss=0.653521 avg=0.668914 VRAM=38.6GiB | 15.3% done | ETA(epoch): 928s + [Epoch 76][140/200] loss=0.714569 avg=0.669803 VRAM=38.5GiB | 15.3% done | ETA(epoch): 796s + [Epoch 76][150/200] loss=0.752643 avg=0.670336 VRAM=38.6GiB | 15.3% done | ETA(epoch): 663s + [Epoch 76][160/200] loss=0.584715 avg=0.669735 VRAM=38.5GiB | 15.4% done | ETA(epoch): 531s + [Epoch 76][170/200] loss=0.623934 avg=0.668559 VRAM=38.6GiB | 15.4% done | ETA(epoch): 398s + [Epoch 76][180/200] loss=0.602669 avg=0.669590 VRAM=38.5GiB | 15.4% done | ETA(epoch): 266s + [Epoch 76][190/200] loss=0.590806 avg=0.667700 VRAM=38.6GiB | 15.4% done | ETA(epoch): 133s + [Epoch 76][200/200] loss=0.736962 avg=0.669470 VRAM=38.5GiB | 15.4% done | ETA(epoch): 0s + Train loss: 0.669470 (2655.6s) ETA: 20855min + Val loss: 0.683724 [t_0.0-0.2=0.9656 t_0.2-0.4=0.7691 t_0.4-0.6=0.6218 t_0.6-0.8=0.5632 t_0.8-1.0=0.5779] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 77/499 (15% done) --- + [Epoch 77][10/200] loss=0.683161 avg=0.696543 VRAM=38.6GiB | 15.4% done | ETA(epoch): 2522s + [Epoch 77][20/200] loss=0.670041 avg=0.681634 VRAM=38.5GiB | 15.4% done | ETA(epoch): 2388s + [Epoch 77][30/200] loss=0.744915 avg=0.686122 VRAM=38.6GiB | 15.4% done | ETA(epoch): 2255s + [Epoch 77][40/200] loss=0.541626 avg=0.676154 VRAM=38.5GiB | 15.4% done | ETA(epoch): 2123s + [Epoch 77][50/200] loss=0.622223 avg=0.672832 VRAM=38.6GiB | 15.4% done | ETA(epoch): 1991s + [Epoch 77][60/200] loss=0.717573 avg=0.669449 VRAM=38.5GiB | 15.5% done | ETA(epoch): 1858s + [Epoch 77][70/200] loss=0.750944 avg=0.672753 VRAM=38.6GiB | 15.5% done | ETA(epoch): 1725s + [Epoch 77][80/200] loss=0.646298 avg=0.674144 VRAM=38.5GiB | 15.5% done | ETA(epoch): 1592s + [Epoch 77][90/200] loss=0.591933 avg=0.670974 VRAM=38.6GiB | 15.5% done | ETA(epoch): 1460s + [Epoch 77][100/200] loss=0.768997 avg=0.666080 VRAM=38.5GiB | 15.5% done | ETA(epoch): 1327s + [Epoch 77][110/200] loss=0.659755 avg=0.664543 VRAM=38.6GiB | 15.5% done | ETA(epoch): 1195s + [Epoch 77][120/200] loss=0.544967 avg=0.664710 VRAM=38.5GiB | 15.5% done | ETA(epoch): 1062s + [Epoch 77][130/200] loss=0.543818 avg=0.662869 VRAM=38.6GiB | 15.5% done | ETA(epoch): 929s + [Epoch 77][140/200] loss=0.595327 avg=0.660911 VRAM=38.5GiB | 15.5% done | ETA(epoch): 796s + [Epoch 77][150/200] loss=0.647927 avg=0.658563 VRAM=38.6GiB | 15.6% done | ETA(epoch): 664s + [Epoch 77][160/200] loss=0.692002 avg=0.659571 VRAM=38.5GiB | 15.6% done | ETA(epoch): 531s + [Epoch 77][170/200] loss=0.845890 avg=0.660354 VRAM=38.6GiB | 15.6% done | ETA(epoch): 398s + [Epoch 77][180/200] loss=0.693736 avg=0.662403 VRAM=38.5GiB | 15.6% done | ETA(epoch): 265s + [Epoch 77][190/200] loss=0.682847 avg=0.662345 VRAM=38.6GiB | 15.6% done | ETA(epoch): 133s + [Epoch 77][200/200] loss=0.594795 avg=0.662553 VRAM=38.5GiB | 15.6% done | ETA(epoch): 0s + Train loss: 0.662553 (2653.6s) ETA: 20800min + Val loss: 0.701199 [t_0.0-0.2=0.9494 t_0.2-0.4=0.7487 t_0.4-0.6=0.6312 t_0.6-0.8=0.5775 t_0.8-1.0=0.5743] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 78/499 (16% done) --- + [Epoch 78][10/200] loss=0.732313 avg=0.655801 VRAM=38.6GiB | 15.6% done | ETA(epoch): 2518s + [Epoch 78][20/200] loss=0.564030 avg=0.659982 VRAM=38.5GiB | 15.6% done | ETA(epoch): 2384s + [Epoch 78][30/200] loss=0.798815 avg=0.661382 VRAM=38.6GiB | 15.6% done | ETA(epoch): 2253s + [Epoch 78][40/200] loss=0.591417 avg=0.661692 VRAM=38.5GiB | 15.6% done | ETA(epoch): 2120s + [Epoch 78][50/200] loss=0.766011 avg=0.660941 VRAM=38.6GiB | 15.7% done | ETA(epoch): 1989s + [Epoch 78][60/200] loss=0.635704 avg=0.659650 VRAM=38.5GiB | 15.7% done | ETA(epoch): 1856s + [Epoch 78][70/200] loss=0.754754 avg=0.663221 VRAM=38.6GiB | 15.7% done | ETA(epoch): 1724s + [Epoch 78][80/200] loss=0.692037 avg=0.669086 VRAM=38.5GiB | 15.7% done | ETA(epoch): 1591s + [Epoch 78][90/200] loss=0.637927 avg=0.670574 VRAM=38.6GiB | 15.7% done | ETA(epoch): 1458s + [Epoch 78][100/200] loss=0.844448 avg=0.670426 VRAM=38.5GiB | 15.7% done | ETA(epoch): 1326s + [Epoch 78][110/200] loss=0.658505 avg=0.668783 VRAM=38.6GiB | 15.7% done | ETA(epoch): 1194s + [Epoch 78][120/200] loss=0.664487 avg=0.668429 VRAM=38.5GiB | 15.7% done | ETA(epoch): 1061s + [Epoch 78][130/200] loss=0.652288 avg=0.668994 VRAM=38.6GiB | 15.7% done | ETA(epoch): 928s + [Epoch 78][140/200] loss=0.787835 avg=0.667292 VRAM=38.5GiB | 15.7% done | ETA(epoch): 796s + [Epoch 78][150/200] loss=0.648243 avg=0.668733 VRAM=38.6GiB | 15.8% done | ETA(epoch): 663s + [Epoch 78][160/200] loss=0.652397 avg=0.667450 VRAM=38.5GiB | 15.8% done | ETA(epoch): 530s + [Epoch 78][170/200] loss=0.736354 avg=0.666633 VRAM=38.6GiB | 15.8% done | ETA(epoch): 398s + [Epoch 78][180/200] loss=0.828651 avg=0.667200 VRAM=38.5GiB | 15.8% done | ETA(epoch): 265s + [Epoch 78][190/200] loss=0.598336 avg=0.666979 VRAM=38.6GiB | 15.8% done | ETA(epoch): 133s + [Epoch 78][200/200] loss=0.633905 avg=0.665791 VRAM=38.5GiB | 15.8% done | ETA(epoch): 0s + Train loss: 0.665791 (2659.8s) ETA: 20747min + Val loss: 0.686416 [t_0.0-0.2=0.9132 t_0.2-0.4=0.7441 t_0.4-0.6=0.6383 t_0.6-0.8=0.5713 t_0.8-1.0=0.5683] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 79/499 (16% done) --- + [Epoch 79][10/200] loss=0.680606 avg=0.670702 VRAM=38.6GiB | 15.8% done | ETA(epoch): 2517s + [Epoch 79][20/200] loss=0.717502 avg=0.666969 VRAM=38.5GiB | 15.8% done | ETA(epoch): 2386s + [Epoch 79][30/200] loss=0.633434 avg=0.654723 VRAM=38.6GiB | 15.8% done | ETA(epoch): 2255s + [Epoch 79][40/200] loss=0.737117 avg=0.650422 VRAM=38.5GiB | 15.8% done | ETA(epoch): 2122s + [Epoch 79][50/200] loss=0.752987 avg=0.654529 VRAM=38.6GiB | 15.8% done | ETA(epoch): 1990s + [Epoch 79][60/200] loss=0.797614 avg=0.655987 VRAM=38.5GiB | 15.9% done | ETA(epoch): 1865s + [Epoch 79][70/200] loss=0.751265 avg=0.659863 VRAM=38.6GiB | 15.9% done | ETA(epoch): 1731s + [Epoch 79][80/200] loss=0.748748 avg=0.662400 VRAM=38.5GiB | 15.9% done | ETA(epoch): 1597s + [Epoch 79][90/200] loss=0.709753 avg=0.661844 VRAM=38.6GiB | 15.9% done | ETA(epoch): 1463s + [Epoch 79][100/200] loss=0.750772 avg=0.662877 VRAM=38.5GiB | 15.9% done | ETA(epoch): 1330s + [Epoch 79][110/200] loss=0.614956 avg=0.659203 VRAM=38.6GiB | 15.9% done | ETA(epoch): 1196s + [Epoch 79][120/200] loss=0.672057 avg=0.659866 VRAM=38.5GiB | 15.9% done | ETA(epoch): 1063s + [Epoch 79][130/200] loss=0.620390 avg=0.660474 VRAM=38.6GiB | 15.9% done | ETA(epoch): 930s + [Epoch 79][140/200] loss=0.712804 avg=0.659538 VRAM=38.5GiB | 15.9% done | ETA(epoch): 797s + [Epoch 79][150/200] loss=0.648034 avg=0.658915 VRAM=38.6GiB | 16.0% done | ETA(epoch): 665s + [Epoch 79][160/200] loss=0.734813 avg=0.657704 VRAM=38.5GiB | 16.0% done | ETA(epoch): 532s + [Epoch 79][170/200] loss=0.908340 avg=0.659370 VRAM=38.6GiB | 16.0% done | ETA(epoch): 399s + [Epoch 79][180/200] loss=0.749250 avg=0.660288 VRAM=38.5GiB | 16.0% done | ETA(epoch): 266s + [Epoch 79][190/200] loss=0.678202 avg=0.662110 VRAM=38.6GiB | 16.0% done | ETA(epoch): 133s + [Epoch 79][200/200] loss=0.643866 avg=0.663494 VRAM=38.5GiB | 16.0% done | ETA(epoch): 0s + Train loss: 0.663494 (2658.5s) ETA: 20693min + Val loss: 0.698701 [t_0.0-0.2=0.9279 t_0.2-0.4=0.7620 t_0.4-0.6=0.6296 t_0.6-0.8=0.5681 t_0.8-1.0=0.5752] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 80/499 (16% done) --- + [Epoch 80][10/200] loss=0.605958 avg=0.689491 VRAM=38.6GiB | 16.0% done | ETA(epoch): 2526s + [Epoch 80][20/200] loss=0.693066 avg=0.656019 VRAM=38.5GiB | 16.0% done | ETA(epoch): 2389s + [Epoch 80][30/200] loss=0.609239 avg=0.642187 VRAM=38.6GiB | 16.0% done | ETA(epoch): 2258s + [Epoch 80][40/200] loss=0.567505 avg=0.641210 VRAM=38.5GiB | 16.0% done | ETA(epoch): 2123s + [Epoch 80][50/200] loss=0.646990 avg=0.646716 VRAM=38.6GiB | 16.1% done | ETA(epoch): 1990s + [Epoch 80][60/200] loss=0.559769 avg=0.648048 VRAM=38.5GiB | 16.1% done | ETA(epoch): 1857s + [Epoch 80][70/200] loss=0.705489 avg=0.648038 VRAM=38.6GiB | 16.1% done | ETA(epoch): 1724s + [Epoch 80][80/200] loss=0.590133 avg=0.652201 VRAM=38.5GiB | 16.1% done | ETA(epoch): 1592s + [Epoch 80][90/200] loss=0.608176 avg=0.656624 VRAM=38.6GiB | 16.1% done | ETA(epoch): 1460s + [Epoch 80][100/200] loss=0.594476 avg=0.657166 VRAM=38.5GiB | 16.1% done | ETA(epoch): 1327s + [Epoch 80][110/200] loss=0.639562 avg=0.657684 VRAM=38.6GiB | 16.1% done | ETA(epoch): 1194s + [Epoch 80][120/200] loss=0.600627 avg=0.656664 VRAM=38.5GiB | 16.1% done | ETA(epoch): 1062s + [Epoch 80][130/200] loss=0.831663 avg=0.658177 VRAM=38.6GiB | 16.1% done | ETA(epoch): 929s + [Epoch 80][140/200] loss=0.605865 avg=0.658862 VRAM=38.5GiB | 16.1% done | ETA(epoch): 796s + [Epoch 80][150/200] loss=0.684116 avg=0.658107 VRAM=38.6GiB | 16.2% done | ETA(epoch): 664s + [Epoch 80][160/200] loss=0.615650 avg=0.657848 VRAM=38.5GiB | 16.2% done | ETA(epoch): 531s + [Epoch 80][170/200] loss=0.616204 avg=0.657936 VRAM=38.6GiB | 16.2% done | ETA(epoch): 398s + [Epoch 80][180/200] loss=0.725015 avg=0.658537 VRAM=38.5GiB | 16.2% done | ETA(epoch): 265s + [Epoch 80][190/200] loss=0.645927 avg=0.657863 VRAM=38.6GiB | 16.2% done | ETA(epoch): 133s + [Epoch 80][200/200] loss=0.624014 avg=0.657492 VRAM=38.5GiB | 16.2% done | ETA(epoch): 0s + Train loss: 0.657492 (2653.9s) ETA: 20639min + Val loss: 0.690709 [t_0.0-0.2=0.9264 t_0.2-0.4=0.7511 t_0.4-0.6=0.6517 t_0.6-0.8=0.5582 t_0.8-1.0=0.5825] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 81/499 (16% done) --- + [Epoch 81][10/200] loss=0.579144 avg=0.669330 VRAM=38.6GiB | 16.2% done | ETA(epoch): 2516s + [Epoch 81][20/200] loss=0.588181 avg=0.659488 VRAM=38.5GiB | 16.2% done | ETA(epoch): 2387s + [Epoch 81][30/200] loss=0.631903 avg=0.665842 VRAM=38.6GiB | 16.2% done | ETA(epoch): 2254s + [Epoch 81][40/200] loss=0.621460 avg=0.669041 VRAM=38.5GiB | 16.2% done | ETA(epoch): 2122s + [Epoch 81][50/200] loss=0.586156 avg=0.662301 VRAM=38.6GiB | 16.2% done | ETA(epoch): 1989s + [Epoch 81][60/200] loss=0.730249 avg=0.661529 VRAM=38.5GiB | 16.3% done | ETA(epoch): 1857s + [Epoch 81][70/200] loss=0.724167 avg=0.662978 VRAM=38.6GiB | 16.3% done | ETA(epoch): 1724s + [Epoch 81][80/200] loss=0.736138 avg=0.658131 VRAM=38.5GiB | 16.3% done | ETA(epoch): 1593s + [Epoch 81][90/200] loss=0.622905 avg=0.657513 VRAM=38.6GiB | 16.3% done | ETA(epoch): 1460s + [Epoch 81][100/200] loss=0.835445 avg=0.665033 VRAM=38.5GiB | 16.3% done | ETA(epoch): 1327s + [Epoch 81][110/200] loss=0.730527 avg=0.663872 VRAM=38.6GiB | 16.3% done | ETA(epoch): 1194s + [Epoch 81][120/200] loss=0.616600 avg=0.663741 VRAM=38.5GiB | 16.3% done | ETA(epoch): 1062s + [Epoch 81][130/200] loss=0.627476 avg=0.664498 VRAM=38.6GiB | 16.3% done | ETA(epoch): 929s + [Epoch 81][140/200] loss=0.566760 avg=0.665655 VRAM=38.5GiB | 16.3% done | ETA(epoch): 796s + [Epoch 81][150/200] loss=0.649347 avg=0.668093 VRAM=38.6GiB | 16.4% done | ETA(epoch): 664s + [Epoch 81][160/200] loss=0.651713 avg=0.665395 VRAM=38.5GiB | 16.4% done | ETA(epoch): 531s + [Epoch 81][170/200] loss=0.678401 avg=0.665503 VRAM=38.6GiB | 16.4% done | ETA(epoch): 398s + [Epoch 81][180/200] loss=0.505823 avg=0.665872 VRAM=38.5GiB | 16.4% done | ETA(epoch): 265s + [Epoch 81][190/200] loss=0.664909 avg=0.664811 VRAM=38.6GiB | 16.4% done | ETA(epoch): 133s + [Epoch 81][200/200] loss=0.681466 avg=0.665520 VRAM=38.5GiB | 16.4% done | ETA(epoch): 0s + Train loss: 0.665520 (2654.2s) ETA: 20586min + Val loss: 0.688065 [t_0.0-0.2=0.9531 t_0.2-0.4=0.7574 t_0.4-0.6=0.6391 t_0.6-0.8=0.5522 t_0.8-1.0=0.5857] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 21.8/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 82/499 (16% done) --- + [Epoch 82][10/200] loss=0.603588 avg=0.661738 VRAM=38.6GiB | 16.4% done | ETA(epoch): 2522s + [Epoch 82][20/200] loss=0.533300 avg=0.649743 VRAM=38.5GiB | 16.4% done | ETA(epoch): 2388s + [Epoch 82][30/200] loss=0.568839 avg=0.649701 VRAM=38.6GiB | 16.4% done | ETA(epoch): 2256s + [Epoch 82][40/200] loss=0.763369 avg=0.644909 VRAM=38.5GiB | 16.4% done | ETA(epoch): 2132s + [Epoch 82][50/200] loss=0.553978 avg=0.651667 VRAM=38.6GiB | 16.4% done | ETA(epoch): 1997s + [Epoch 82][60/200] loss=0.631483 avg=0.653115 VRAM=38.5GiB | 16.5% done | ETA(epoch): 1863s + [Epoch 82][70/200] loss=0.593879 avg=0.656254 VRAM=38.6GiB | 16.5% done | ETA(epoch): 1729s + [Epoch 82][80/200] loss=0.626042 avg=0.656028 VRAM=38.5GiB | 16.5% done | ETA(epoch): 1596s + [Epoch 82][90/200] loss=0.603584 avg=0.657688 VRAM=38.6GiB | 16.5% done | ETA(epoch): 1462s + [Epoch 82][100/200] loss=0.710025 avg=0.658002 VRAM=38.5GiB | 16.5% done | ETA(epoch): 1329s + [Epoch 82][110/200] loss=0.730820 avg=0.660350 VRAM=38.6GiB | 16.5% done | ETA(epoch): 1196s + [Epoch 82][120/200] loss=0.627902 avg=0.660646 VRAM=38.5GiB | 16.5% done | ETA(epoch): 1063s + [Epoch 82][130/200] loss=0.649892 avg=0.660231 VRAM=38.6GiB | 16.5% done | ETA(epoch): 933s + [Epoch 82][140/200] loss=0.748708 avg=0.660681 VRAM=38.5GiB | 16.5% done | ETA(epoch): 799s + [Epoch 82][150/200] loss=0.774528 avg=0.661539 VRAM=38.6GiB | 16.6% done | ETA(epoch): 666s + [Epoch 82][160/200] loss=0.579896 avg=0.661844 VRAM=38.5GiB | 16.6% done | ETA(epoch): 533s + [Epoch 82][170/200] loss=0.762680 avg=0.664584 VRAM=38.6GiB | 16.6% done | ETA(epoch): 399s + [Epoch 82][180/200] loss=0.681199 avg=0.662600 VRAM=38.5GiB | 16.6% done | ETA(epoch): 266s + [Epoch 82][190/200] loss=0.556313 avg=0.662838 VRAM=38.6GiB | 16.6% done | ETA(epoch): 133s + [Epoch 82][200/200] loss=0.728330 avg=0.663080 VRAM=38.5GiB | 16.6% done | ETA(epoch): 0s + Train loss: 0.663080 (2661.6s) ETA: 20533min + Val loss: 0.696454 [t_0.0-0.2=0.9384 t_0.2-0.4=0.7430 t_0.4-0.6=0.6286 t_0.6-0.8=0.5807 t_0.8-1.0=0.5818] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 83/499 (17% done) --- + [Epoch 83][10/200] loss=0.699849 avg=0.630920 VRAM=38.6GiB | 16.6% done | ETA(epoch): 2521s + [Epoch 83][20/200] loss=0.719772 avg=0.635281 VRAM=38.5GiB | 16.6% done | ETA(epoch): 2387s + [Epoch 83][30/200] loss=0.586623 avg=0.639494 VRAM=38.6GiB | 16.6% done | ETA(epoch): 2255s + [Epoch 83][40/200] loss=0.572934 avg=0.635807 VRAM=38.5GiB | 16.6% done | ETA(epoch): 2123s + [Epoch 83][50/200] loss=0.687013 avg=0.641767 VRAM=38.6GiB | 16.7% done | ETA(epoch): 1990s + [Epoch 83][60/200] loss=0.627034 avg=0.642679 VRAM=38.5GiB | 16.7% done | ETA(epoch): 1858s + [Epoch 83][70/200] loss=0.602800 avg=0.646150 VRAM=38.6GiB | 16.7% done | ETA(epoch): 1726s + [Epoch 83][80/200] loss=0.754713 avg=0.646667 VRAM=38.5GiB | 16.7% done | ETA(epoch): 1593s + [Epoch 83][90/200] loss=0.712867 avg=0.647804 VRAM=38.6GiB | 16.7% done | ETA(epoch): 1469s + [Epoch 83][100/200] loss=0.723306 avg=0.653450 VRAM=38.5GiB | 16.7% done | ETA(epoch): 1334s + [Epoch 83][110/200] loss=0.641018 avg=0.654218 VRAM=38.6GiB | 16.7% done | ETA(epoch): 1200s + [Epoch 83][120/200] loss=0.635836 avg=0.654683 VRAM=38.5GiB | 16.7% done | ETA(epoch): 1066s + [Epoch 83][130/200] loss=0.723055 avg=0.656531 VRAM=38.6GiB | 16.7% done | ETA(epoch): 933s + [Epoch 83][140/200] loss=0.689848 avg=0.654996 VRAM=38.5GiB | 16.7% done | ETA(epoch): 799s + [Epoch 83][150/200] loss=0.760740 avg=0.655872 VRAM=38.6GiB | 16.8% done | ETA(epoch): 666s + [Epoch 83][160/200] loss=0.699420 avg=0.658054 VRAM=38.5GiB | 16.8% done | ETA(epoch): 533s + [Epoch 83][170/200] loss=0.737441 avg=0.658945 VRAM=38.6GiB | 16.8% done | ETA(epoch): 399s + [Epoch 83][180/200] loss=0.660556 avg=0.658390 VRAM=38.5GiB | 16.8% done | ETA(epoch): 266s + [Epoch 83][190/200] loss=0.668322 avg=0.658896 VRAM=38.6GiB | 16.8% done | ETA(epoch): 133s + [Epoch 83][200/200] loss=0.634700 avg=0.657650 VRAM=38.5GiB | 16.8% done | ETA(epoch): 0s + Train loss: 0.657650 (2663.7s) ETA: 20480min + Val loss: 0.684862 [t_0.0-0.2=0.9232 t_0.2-0.4=0.7584 t_0.4-0.6=0.6369 t_0.6-0.8=0.5573 t_0.8-1.0=0.5719] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 22.5/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 84/499 (17% done) --- + [Epoch 84][10/200] loss=0.664876 avg=0.640570 VRAM=38.6GiB | 16.8% done | ETA(epoch): 2521s + [Epoch 84][20/200] loss=0.820917 avg=0.657345 VRAM=38.5GiB | 16.8% done | ETA(epoch): 2389s + [Epoch 84][30/200] loss=0.731869 avg=0.657228 VRAM=38.6GiB | 16.8% done | ETA(epoch): 2257s + [Epoch 84][40/200] loss=0.678643 avg=0.651357 VRAM=38.5GiB | 16.8% done | ETA(epoch): 2124s + [Epoch 84][50/200] loss=0.803627 avg=0.654640 VRAM=38.6GiB | 16.9% done | ETA(epoch): 1991s + [Epoch 84][60/200] loss=0.677904 avg=0.655340 VRAM=38.5GiB | 16.9% done | ETA(epoch): 1864s + [Epoch 84][70/200] loss=0.666226 avg=0.662980 VRAM=38.6GiB | 16.9% done | ETA(epoch): 1730s + [Epoch 84][80/200] loss=0.589252 avg=0.662867 VRAM=38.5GiB | 16.9% done | ETA(epoch): 1597s + [Epoch 84][90/200] loss=0.698896 avg=0.664676 VRAM=38.6GiB | 16.9% done | ETA(epoch): 1463s + [Epoch 84][100/200] loss=0.692020 avg=0.665805 VRAM=38.5GiB | 16.9% done | ETA(epoch): 1330s + [Epoch 84][110/200] loss=0.625599 avg=0.664218 VRAM=38.6GiB | 16.9% done | ETA(epoch): 1197s + [Epoch 84][120/200] loss=0.962303 avg=0.666811 VRAM=38.5GiB | 16.9% done | ETA(epoch): 1064s + [Epoch 84][130/200] loss=0.695532 avg=0.668567 VRAM=38.6GiB | 16.9% done | ETA(epoch): 931s + [Epoch 84][140/200] loss=0.606578 avg=0.666490 VRAM=38.5GiB | 16.9% done | ETA(epoch): 797s + [Epoch 84][150/200] loss=0.666287 avg=0.665402 VRAM=38.6GiB | 17.0% done | ETA(epoch): 665s + [Epoch 84][160/200] loss=0.490195 avg=0.660359 VRAM=38.5GiB | 17.0% done | ETA(epoch): 532s + [Epoch 84][170/200] loss=0.827979 avg=0.661934 VRAM=38.6GiB | 17.0% done | ETA(epoch): 399s + [Epoch 84][180/200] loss=0.649489 avg=0.665075 VRAM=38.5GiB | 17.0% done | ETA(epoch): 266s + [Epoch 84][190/200] loss=0.563512 avg=0.665495 VRAM=38.6GiB | 17.0% done | ETA(epoch): 133s + [Epoch 84][200/200] loss=0.635027 avg=0.664461 VRAM=38.5GiB | 17.0% done | ETA(epoch): 0s + Train loss: 0.664461 (2659.5s) ETA: 20427min + Val loss: 0.707101 [t_0.0-0.2=0.9353 t_0.2-0.4=0.7509 t_0.4-0.6=0.6310 t_0.6-0.8=0.5651 t_0.8-1.0=0.5912] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 85/499 (17% done) --- + [Epoch 85][10/200] loss=0.611286 avg=0.675293 VRAM=38.6GiB | 17.0% done | ETA(epoch): 2525s + [Epoch 85][20/200] loss=0.778612 avg=0.670983 VRAM=38.5GiB | 17.0% done | ETA(epoch): 2388s + [Epoch 85][30/200] loss=0.562032 avg=0.660034 VRAM=38.6GiB | 17.0% done | ETA(epoch): 2256s + [Epoch 85][40/200] loss=0.698706 avg=0.659721 VRAM=38.5GiB | 17.0% done | ETA(epoch): 2122s + [Epoch 85][50/200] loss=0.575823 avg=0.664652 VRAM=38.6GiB | 17.1% done | ETA(epoch): 1991s + [Epoch 85][60/200] loss=0.648283 avg=0.661547 VRAM=38.5GiB | 17.1% done | ETA(epoch): 1858s + [Epoch 85][70/200] loss=0.619107 avg=0.662992 VRAM=38.6GiB | 17.1% done | ETA(epoch): 1725s + [Epoch 85][80/200] loss=0.631567 avg=0.662344 VRAM=38.5GiB | 17.1% done | ETA(epoch): 1592s + [Epoch 85][90/200] loss=0.600923 avg=0.661004 VRAM=38.6GiB | 17.1% done | ETA(epoch): 1459s + [Epoch 85][100/200] loss=0.762604 avg=0.664028 VRAM=38.5GiB | 17.1% done | ETA(epoch): 1327s + [Epoch 85][110/200] loss=0.815552 avg=0.666369 VRAM=38.6GiB | 17.1% done | ETA(epoch): 1195s + [Epoch 85][120/200] loss=0.568474 avg=0.665205 VRAM=38.5GiB | 17.1% done | ETA(epoch): 1062s + [Epoch 85][130/200] loss=0.542125 avg=0.663742 VRAM=38.6GiB | 17.1% done | ETA(epoch): 929s + [Epoch 85][140/200] loss=0.658427 avg=0.665348 VRAM=38.5GiB | 17.1% done | ETA(epoch): 797s + [Epoch 85][150/200] loss=0.578193 avg=0.663532 VRAM=38.6GiB | 17.2% done | ETA(epoch): 664s + [Epoch 85][160/200] loss=0.818990 avg=0.663418 VRAM=38.5GiB | 17.2% done | ETA(epoch): 531s + [Epoch 85][170/200] loss=0.674671 avg=0.665432 VRAM=38.6GiB | 17.2% done | ETA(epoch): 398s + [Epoch 85][180/200] loss=0.811950 avg=0.666380 VRAM=38.5GiB | 17.2% done | ETA(epoch): 265s + [Epoch 85][190/200] loss=0.595435 avg=0.664978 VRAM=38.6GiB | 17.2% done | ETA(epoch): 133s + [Epoch 85][200/200] loss=0.714694 avg=0.664830 VRAM=38.5GiB | 17.2% done | ETA(epoch): 0s + Train loss: 0.664830 (2654.4s) ETA: 20374min + Val loss: 0.688241 [t_0.0-0.2=0.9228 t_0.2-0.4=0.7490 t_0.4-0.6=0.6643 t_0.6-0.8=0.5555 t_0.8-1.0=0.5781] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 86/499 (17% done) --- + [Epoch 86][10/200] loss=0.579858 avg=0.624304 VRAM=38.6GiB | 17.2% done | ETA(epoch): 2516s + [Epoch 86][20/200] loss=0.833936 avg=0.645287 VRAM=38.5GiB | 17.2% done | ETA(epoch): 2388s + [Epoch 86][30/200] loss=0.661522 avg=0.654351 VRAM=38.6GiB | 17.2% done | ETA(epoch): 2255s + [Epoch 86][40/200] loss=0.702162 avg=0.650288 VRAM=38.5GiB | 17.2% done | ETA(epoch): 2124s + [Epoch 86][50/200] loss=0.707285 avg=0.656846 VRAM=38.6GiB | 17.2% done | ETA(epoch): 1991s + [Epoch 86][60/200] loss=0.653984 avg=0.658214 VRAM=38.5GiB | 17.3% done | ETA(epoch): 1858s + [Epoch 86][70/200] loss=0.840514 avg=0.663579 VRAM=38.6GiB | 17.3% done | ETA(epoch): 1725s + [Epoch 86][80/200] loss=0.569301 avg=0.657833 VRAM=38.5GiB | 17.3% done | ETA(epoch): 1592s + [Epoch 86][90/200] loss=0.698869 avg=0.658611 VRAM=38.6GiB | 17.3% done | ETA(epoch): 1460s + [Epoch 86][100/200] loss=0.638394 avg=0.658021 VRAM=38.5GiB | 17.3% done | ETA(epoch): 1327s + [Epoch 86][110/200] loss=0.633865 avg=0.658040 VRAM=38.6GiB | 17.3% done | ETA(epoch): 1195s + [Epoch 86][120/200] loss=0.606760 avg=0.655969 VRAM=38.5GiB | 17.3% done | ETA(epoch): 1062s + [Epoch 86][130/200] loss=0.732138 avg=0.654806 VRAM=38.6GiB | 17.3% done | ETA(epoch): 929s + [Epoch 86][140/200] loss=0.611197 avg=0.658237 VRAM=38.5GiB | 17.3% done | ETA(epoch): 796s + [Epoch 86][150/200] loss=0.605935 avg=0.658033 VRAM=38.6GiB | 17.3% done | ETA(epoch): 664s + [Epoch 86][160/200] loss=0.632172 avg=0.659108 VRAM=38.5GiB | 17.4% done | ETA(epoch): 531s + [Epoch 86][170/200] loss=0.687569 avg=0.660180 VRAM=38.6GiB | 17.4% done | ETA(epoch): 399s + [Epoch 86][180/200] loss=0.681757 avg=0.661466 VRAM=38.5GiB | 17.4% done | ETA(epoch): 266s + [Epoch 86][190/200] loss=0.656955 avg=0.662292 VRAM=38.6GiB | 17.4% done | ETA(epoch): 133s + [Epoch 86][200/200] loss=0.561966 avg=0.663802 VRAM=38.5GiB | 17.4% done | ETA(epoch): 0s + Train loss: 0.663802 (2656.9s) ETA: 20321min + Val loss: 0.707359 [t_0.0-0.2=0.9495 t_0.2-0.4=0.7601 t_0.4-0.6=0.6316 t_0.6-0.8=0.5554 t_0.8-1.0=0.5784] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 87/499 (17% done) --- + [Epoch 87][10/200] loss=0.581414 avg=0.671199 VRAM=38.6GiB | 17.4% done | ETA(epoch): 2523s + [Epoch 87][20/200] loss=0.594183 avg=0.683010 VRAM=38.5GiB | 17.4% done | ETA(epoch): 2389s + [Epoch 87][30/200] loss=0.583376 avg=0.672759 VRAM=38.6GiB | 17.4% done | ETA(epoch): 2256s + [Epoch 87][40/200] loss=0.665237 avg=0.664629 VRAM=38.5GiB | 17.4% done | ETA(epoch): 2133s + [Epoch 87][50/200] loss=0.686577 avg=0.664623 VRAM=38.6GiB | 17.4% done | ETA(epoch): 1998s + [Epoch 87][60/200] loss=0.610403 avg=0.661522 VRAM=38.5GiB | 17.5% done | ETA(epoch): 1864s + [Epoch 87][70/200] loss=0.688814 avg=0.663740 VRAM=38.6GiB | 17.5% done | ETA(epoch): 1730s + [Epoch 87][80/200] loss=0.735258 avg=0.671056 VRAM=38.5GiB | 17.5% done | ETA(epoch): 1596s + [Epoch 87][90/200] loss=0.605300 avg=0.669402 VRAM=38.6GiB | 17.5% done | ETA(epoch): 1463s + [Epoch 87][100/200] loss=0.620422 avg=0.667977 VRAM=38.5GiB | 17.5% done | ETA(epoch): 1330s + [Epoch 87][110/200] loss=0.671103 avg=0.666654 VRAM=38.6GiB | 17.5% done | ETA(epoch): 1197s + [Epoch 87][120/200] loss=0.652622 avg=0.666247 VRAM=38.5GiB | 17.5% done | ETA(epoch): 1064s + [Epoch 87][130/200] loss=0.551922 avg=0.666331 VRAM=38.6GiB | 17.5% done | ETA(epoch): 931s + [Epoch 87][140/200] loss=0.692938 avg=0.666891 VRAM=38.5GiB | 17.5% done | ETA(epoch): 797s + [Epoch 87][150/200] loss=0.589343 avg=0.665252 VRAM=38.6GiB | 17.5% done | ETA(epoch): 665s + [Epoch 87][160/200] loss=0.726383 avg=0.665569 VRAM=38.5GiB | 17.6% done | ETA(epoch): 532s + [Epoch 87][170/200] loss=0.591657 avg=0.663748 VRAM=38.6GiB | 17.6% done | ETA(epoch): 399s + [Epoch 87][180/200] loss=0.563818 avg=0.662248 VRAM=38.5GiB | 17.6% done | ETA(epoch): 266s + [Epoch 87][190/200] loss=0.731753 avg=0.664343 VRAM=38.6GiB | 17.6% done | ETA(epoch): 133s + [Epoch 87][200/200] loss=0.644843 avg=0.665397 VRAM=38.5GiB | 17.6% done | ETA(epoch): 0s + Train loss: 0.665397 (2657.8s) ETA: 20269min + Val loss: 0.676321 [t_0.0-0.2=0.9099 t_0.2-0.4=0.7629 t_0.4-0.6=0.6340 t_0.6-0.8=0.5564 t_0.8-1.0=0.5612] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0087 (BEST) + Deleted old checkpoint: checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 88/499 (18% done) --- + [Epoch 88][10/200] loss=0.793002 avg=0.664183 VRAM=38.6GiB | 17.6% done | ETA(epoch): 2522s + [Epoch 88][20/200] loss=0.658384 avg=0.682712 VRAM=38.5GiB | 17.6% done | ETA(epoch): 2388s + [Epoch 88][30/200] loss=0.672010 avg=0.680418 VRAM=38.6GiB | 17.6% done | ETA(epoch): 2255s + [Epoch 88][40/200] loss=0.717133 avg=0.676664 VRAM=38.5GiB | 17.6% done | ETA(epoch): 2123s + [Epoch 88][50/200] loss=0.728532 avg=0.664794 VRAM=38.6GiB | 17.6% done | ETA(epoch): 1990s + [Epoch 88][60/200] loss=0.626839 avg=0.666134 VRAM=38.5GiB | 17.7% done | ETA(epoch): 1858s + [Epoch 88][70/200] loss=0.711730 avg=0.663597 VRAM=38.6GiB | 17.7% done | ETA(epoch): 1725s + [Epoch 88][80/200] loss=0.623298 avg=0.663364 VRAM=38.5GiB | 17.7% done | ETA(epoch): 1592s + [Epoch 88][90/200] loss=0.647450 avg=0.662223 VRAM=38.6GiB | 17.7% done | ETA(epoch): 1463s + [Epoch 88][100/200] loss=0.718023 avg=0.662889 VRAM=38.5GiB | 17.7% done | ETA(epoch): 1330s + [Epoch 88][110/200] loss=0.673683 avg=0.660956 VRAM=38.6GiB | 17.7% done | ETA(epoch): 1197s + [Epoch 88][120/200] loss=0.606810 avg=0.656374 VRAM=38.5GiB | 17.7% done | ETA(epoch): 1064s + [Epoch 88][130/200] loss=0.600461 avg=0.657448 VRAM=38.6GiB | 17.7% done | ETA(epoch): 931s + [Epoch 88][140/200] loss=0.713087 avg=0.659008 VRAM=38.5GiB | 17.7% done | ETA(epoch): 798s + [Epoch 88][150/200] loss=0.583889 avg=0.659034 VRAM=38.6GiB | 17.8% done | ETA(epoch): 665s + [Epoch 88][160/200] loss=0.559214 avg=0.657863 VRAM=38.5GiB | 17.8% done | ETA(epoch): 532s + [Epoch 88][170/200] loss=0.676036 avg=0.658272 VRAM=38.6GiB | 17.8% done | ETA(epoch): 399s + [Epoch 88][180/200] loss=0.577517 avg=0.656989 VRAM=38.5GiB | 17.8% done | ETA(epoch): 266s + [Epoch 88][190/200] loss=0.699197 avg=0.658867 VRAM=38.6GiB | 17.8% done | ETA(epoch): 133s + [Epoch 88][200/200] loss=0.548838 avg=0.659525 VRAM=38.5GiB | 17.8% done | ETA(epoch): 0s + Train loss: 0.659525 (2657.6s) ETA: 20216min + Val loss: 0.696193 [t_0.0-0.2=0.9371 t_0.2-0.4=0.7579 t_0.4-0.6=0.6309 t_0.6-0.8=0.5590 t_0.8-1.0=0.5805] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 22.3/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 89/499 (18% done) --- + [Epoch 89][10/200] loss=0.611540 avg=0.620654 VRAM=38.6GiB | 17.8% done | ETA(epoch): 2520s + [Epoch 89][20/200] loss=0.702970 avg=0.632704 VRAM=38.5GiB | 17.8% done | ETA(epoch): 2389s + [Epoch 89][30/200] loss=0.623640 avg=0.635780 VRAM=38.6GiB | 17.8% done | ETA(epoch): 2256s + [Epoch 89][40/200] loss=0.517926 avg=0.635129 VRAM=38.5GiB | 17.8% done | ETA(epoch): 2123s + [Epoch 89][50/200] loss=0.622943 avg=0.640187 VRAM=38.6GiB | 17.8% done | ETA(epoch): 1992s + [Epoch 89][60/200] loss=0.642002 avg=0.639236 VRAM=38.5GiB | 17.9% done | ETA(epoch): 1859s + [Epoch 89][70/200] loss=0.594682 avg=0.645677 VRAM=38.6GiB | 17.9% done | ETA(epoch): 1726s + [Epoch 89][80/200] loss=0.673604 avg=0.647445 VRAM=38.5GiB | 17.9% done | ETA(epoch): 1593s + [Epoch 89][90/200] loss=0.711989 avg=0.648682 VRAM=38.6GiB | 17.9% done | ETA(epoch): 1461s + [Epoch 89][100/200] loss=0.688773 avg=0.654867 VRAM=38.5GiB | 17.9% done | ETA(epoch): 1328s + [Epoch 89][110/200] loss=0.590142 avg=0.658290 VRAM=38.6GiB | 17.9% done | ETA(epoch): 1195s + [Epoch 89][120/200] loss=0.566695 avg=0.658329 VRAM=38.5GiB | 17.9% done | ETA(epoch): 1062s + [Epoch 89][130/200] loss=0.646848 avg=0.654975 VRAM=38.6GiB | 17.9% done | ETA(epoch): 930s + [Epoch 89][140/200] loss=0.679599 avg=0.655273 VRAM=38.5GiB | 17.9% done | ETA(epoch): 797s + [Epoch 89][150/200] loss=0.608445 avg=0.655882 VRAM=38.6GiB | 17.9% done | ETA(epoch): 664s + [Epoch 89][160/200] loss=0.613122 avg=0.655224 VRAM=38.5GiB | 18.0% done | ETA(epoch): 531s + [Epoch 89][170/200] loss=0.825631 avg=0.656118 VRAM=38.6GiB | 18.0% done | ETA(epoch): 398s + [Epoch 89][180/200] loss=0.628970 avg=0.657815 VRAM=38.5GiB | 18.0% done | ETA(epoch): 266s + [Epoch 89][190/200] loss=0.659921 avg=0.657402 VRAM=38.6GiB | 18.0% done | ETA(epoch): 133s + [Epoch 89][200/200] loss=0.695008 avg=0.658017 VRAM=38.5GiB | 18.0% done | ETA(epoch): 0s + Train loss: 0.658017 (2655.6s) ETA: 20163min + Val loss: 0.692137 [t_0.0-0.2=0.9332 t_0.2-0.4=0.7471 t_0.4-0.6=0.6433 t_0.6-0.8=0.5754 t_0.8-1.0=0.5659] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 22.3/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 90/499 (18% done) --- + [Epoch 90][10/200] loss=0.722719 avg=0.622841 VRAM=38.6GiB | 18.0% done | ETA(epoch): 2523s + [Epoch 90][20/200] loss=0.692316 avg=0.642184 VRAM=38.5GiB | 18.0% done | ETA(epoch): 2388s + [Epoch 90][30/200] loss=0.610160 avg=0.646069 VRAM=38.6GiB | 18.0% done | ETA(epoch): 2257s + [Epoch 90][40/200] loss=0.582217 avg=0.653677 VRAM=38.5GiB | 18.0% done | ETA(epoch): 2123s + [Epoch 90][50/200] loss=0.624567 avg=0.650672 VRAM=38.6GiB | 18.1% done | ETA(epoch): 1991s + [Epoch 90][60/200] loss=0.624114 avg=0.645879 VRAM=38.5GiB | 18.1% done | ETA(epoch): 1858s + [Epoch 90][70/200] loss=0.670250 avg=0.647126 VRAM=38.6GiB | 18.1% done | ETA(epoch): 1725s + [Epoch 90][80/200] loss=0.751281 avg=0.657658 VRAM=38.5GiB | 18.1% done | ETA(epoch): 1593s + [Epoch 90][90/200] loss=0.709541 avg=0.655649 VRAM=38.6GiB | 18.1% done | ETA(epoch): 1461s + [Epoch 90][100/200] loss=0.845407 avg=0.659111 VRAM=38.5GiB | 18.1% done | ETA(epoch): 1328s + [Epoch 90][110/200] loss=0.699153 avg=0.659244 VRAM=38.6GiB | 18.1% done | ETA(epoch): 1196s + [Epoch 90][120/200] loss=0.675289 avg=0.660115 VRAM=38.5GiB | 18.1% done | ETA(epoch): 1062s + [Epoch 90][130/200] loss=0.578902 avg=0.659250 VRAM=38.6GiB | 18.1% done | ETA(epoch): 930s + [Epoch 90][140/200] loss=0.720778 avg=0.661507 VRAM=38.5GiB | 18.1% done | ETA(epoch): 797s + [Epoch 90][150/200] loss=0.709197 avg=0.664193 VRAM=38.6GiB | 18.1% done | ETA(epoch): 664s + [Epoch 90][160/200] loss=0.711941 avg=0.661373 VRAM=38.5GiB | 18.2% done | ETA(epoch): 531s + [Epoch 90][170/200] loss=0.712359 avg=0.664725 VRAM=38.6GiB | 18.2% done | ETA(epoch): 398s + [Epoch 90][180/200] loss=0.645266 avg=0.664520 VRAM=38.5GiB | 18.2% done | ETA(epoch): 266s + [Epoch 90][190/200] loss=0.719466 avg=0.664780 VRAM=38.6GiB | 18.2% done | ETA(epoch): 133s + [Epoch 90][200/200] loss=0.822213 avg=0.664355 VRAM=38.5GiB | 18.2% done | ETA(epoch): 0s + Train loss: 0.664355 (2655.9s) ETA: 20111min + Val loss: 0.688507 [t_0.0-0.2=0.9176 t_0.2-0.4=0.7647 t_0.4-0.6=0.6524 t_0.6-0.8=0.5671 t_0.8-1.0=0.5690] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0090 +[MEM @ epoch 90 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 91/499 (18% done) --- + [Epoch 91][10/200] loss=0.663091 avg=0.664000 VRAM=38.6GiB | 18.2% done | ETA(epoch): 2518s + [Epoch 91][20/200] loss=0.549330 avg=0.657255 VRAM=38.5GiB | 18.2% done | ETA(epoch): 2389s + [Epoch 91][30/200] loss=0.817967 avg=0.655548 VRAM=38.6GiB | 18.2% done | ETA(epoch): 2255s + [Epoch 91][40/200] loss=0.668714 avg=0.653119 VRAM=38.5GiB | 18.2% done | ETA(epoch): 2123s + [Epoch 91][50/200] loss=0.643030 avg=0.652921 VRAM=38.6GiB | 18.2% done | ETA(epoch): 1990s + [Epoch 91][60/200] loss=0.777211 avg=0.656048 VRAM=38.5GiB | 18.3% done | ETA(epoch): 1857s + [Epoch 91][70/200] loss=0.591505 avg=0.650997 VRAM=38.6GiB | 18.3% done | ETA(epoch): 1725s + [Epoch 91][80/200] loss=0.783034 avg=0.656186 VRAM=38.5GiB | 18.3% done | ETA(epoch): 1592s + [Epoch 91][90/200] loss=0.625143 avg=0.654396 VRAM=38.6GiB | 18.3% done | ETA(epoch): 1460s + [Epoch 91][100/200] loss=0.607731 avg=0.652735 VRAM=38.5GiB | 18.3% done | ETA(epoch): 1327s + [Epoch 91][110/200] loss=0.808623 avg=0.656747 VRAM=38.6GiB | 18.3% done | ETA(epoch): 1194s + [Epoch 91][120/200] loss=0.624084 avg=0.656018 VRAM=38.5GiB | 18.3% done | ETA(epoch): 1062s + [Epoch 91][130/200] loss=0.685794 avg=0.658183 VRAM=38.6GiB | 18.3% done | ETA(epoch): 929s + [Epoch 91][140/200] loss=0.632681 avg=0.658949 VRAM=38.5GiB | 18.3% done | ETA(epoch): 796s + [Epoch 91][150/200] loss=0.647052 avg=0.655989 VRAM=38.6GiB | 18.4% done | ETA(epoch): 664s + [Epoch 91][160/200] loss=0.614180 avg=0.654830 VRAM=38.5GiB | 18.4% done | ETA(epoch): 531s + [Epoch 91][170/200] loss=0.696994 avg=0.654623 VRAM=38.6GiB | 18.4% done | ETA(epoch): 398s + [Epoch 91][180/200] loss=0.674368 avg=0.652943 VRAM=38.5GiB | 18.4% done | ETA(epoch): 265s + [Epoch 91][190/200] loss=0.736682 avg=0.654011 VRAM=38.6GiB | 18.4% done | ETA(epoch): 133s + [Epoch 91][200/200] loss=0.730440 avg=0.654686 VRAM=38.5GiB | 18.4% done | ETA(epoch): 0s + Train loss: 0.654686 (2654.4s) ETA: 20058min + Val loss: 0.699900 [t_0.0-0.2=0.9651 t_0.2-0.4=0.7447 t_0.4-0.6=0.6483 t_0.6-0.8=0.5680 t_0.8-1.0=0.5796] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 22.3/188.5 GiB (11.8%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 92/499 (18% done) --- + [Epoch 92][10/200] loss=0.592354 avg=0.683689 VRAM=38.6GiB | 18.4% done | ETA(epoch): 2525s + [Epoch 92][20/200] loss=0.691835 avg=0.661462 VRAM=38.5GiB | 18.4% done | ETA(epoch): 2390s + [Epoch 92][30/200] loss=0.609035 avg=0.659732 VRAM=38.6GiB | 18.4% done | ETA(epoch): 2272s + [Epoch 92][40/200] loss=0.754898 avg=0.659578 VRAM=38.5GiB | 18.4% done | ETA(epoch): 2135s + [Epoch 92][50/200] loss=0.727568 avg=0.655432 VRAM=38.6GiB | 18.4% done | ETA(epoch): 1999s + [Epoch 92][60/200] loss=0.617650 avg=0.653910 VRAM=38.5GiB | 18.5% done | ETA(epoch): 1865s + [Epoch 92][70/200] loss=0.603813 avg=0.657129 VRAM=38.6GiB | 18.5% done | ETA(epoch): 1730s + [Epoch 92][80/200] loss=0.693980 avg=0.657108 VRAM=38.5GiB | 18.5% done | ETA(epoch): 1597s + [Epoch 92][90/200] loss=0.685593 avg=0.658345 VRAM=38.6GiB | 18.5% done | ETA(epoch): 1463s + [Epoch 92][100/200] loss=0.646747 avg=0.663532 VRAM=38.5GiB | 18.5% done | ETA(epoch): 1330s + [Epoch 92][110/200] loss=0.741681 avg=0.665857 VRAM=38.6GiB | 18.5% done | ETA(epoch): 1197s + [Epoch 92][120/200] loss=0.649433 avg=0.665115 VRAM=38.5GiB | 18.5% done | ETA(epoch): 1066s + [Epoch 92][130/200] loss=0.702389 avg=0.667007 VRAM=38.6GiB | 18.5% done | ETA(epoch): 932s + [Epoch 92][140/200] loss=0.725884 avg=0.667755 VRAM=38.5GiB | 18.5% done | ETA(epoch): 799s + [Epoch 92][150/200] loss=0.659759 avg=0.665611 VRAM=38.6GiB | 18.6% done | ETA(epoch): 666s + [Epoch 92][160/200] loss=0.686487 avg=0.667335 VRAM=38.5GiB | 18.6% done | ETA(epoch): 532s + [Epoch 92][170/200] loss=0.800338 avg=0.668467 VRAM=38.6GiB | 18.6% done | ETA(epoch): 399s + [Epoch 92][180/200] loss=0.756267 avg=0.668661 VRAM=38.5GiB | 18.6% done | ETA(epoch): 266s + [Epoch 92][190/200] loss=0.640674 avg=0.666849 VRAM=38.6GiB | 18.6% done | ETA(epoch): 133s + [Epoch 92][200/200] loss=0.576077 avg=0.665463 VRAM=38.5GiB | 18.6% done | ETA(epoch): 0s + Train loss: 0.665463 (2660.2s) ETA: 20006min + Val loss: 0.696324 [t_0.0-0.2=0.9423 t_0.2-0.4=0.7628 t_0.4-0.6=0.6223 t_0.6-0.8=0.5780 t_0.8-1.0=0.5698] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 93/499 (19% done) --- + [Epoch 93][10/200] loss=0.632336 avg=0.612487 VRAM=38.6GiB | 18.6% done | ETA(epoch): 2519s + [Epoch 93][20/200] loss=0.689252 avg=0.640753 VRAM=38.5GiB | 18.6% done | ETA(epoch): 2388s + [Epoch 93][30/200] loss=0.646931 avg=0.640578 VRAM=38.6GiB | 18.6% done | ETA(epoch): 2257s + [Epoch 93][40/200] loss=0.694028 avg=0.653697 VRAM=38.5GiB | 18.6% done | ETA(epoch): 2124s + [Epoch 93][50/200] loss=0.697713 avg=0.654319 VRAM=38.6GiB | 18.6% done | ETA(epoch): 1991s + [Epoch 93][60/200] loss=0.623151 avg=0.653310 VRAM=38.5GiB | 18.7% done | ETA(epoch): 1858s + [Epoch 93][70/200] loss=0.645910 avg=0.651742 VRAM=38.6GiB | 18.7% done | ETA(epoch): 1726s + [Epoch 93][80/200] loss=0.605941 avg=0.652868 VRAM=38.5GiB | 18.7% done | ETA(epoch): 1597s + [Epoch 93][90/200] loss=0.791260 avg=0.654713 VRAM=38.6GiB | 18.7% done | ETA(epoch): 1463s + [Epoch 93][100/200] loss=0.674764 avg=0.657455 VRAM=38.5GiB | 18.7% done | ETA(epoch): 1330s + [Epoch 93][110/200] loss=0.628259 avg=0.657074 VRAM=38.6GiB | 18.7% done | ETA(epoch): 1197s + [Epoch 93][120/200] loss=0.570716 avg=0.653725 VRAM=38.5GiB | 18.7% done | ETA(epoch): 1063s + [Epoch 93][130/200] loss=0.604773 avg=0.654889 VRAM=38.6GiB | 18.7% done | ETA(epoch): 930s + [Epoch 93][140/200] loss=0.663540 avg=0.653152 VRAM=38.5GiB | 18.7% done | ETA(epoch): 797s + [Epoch 93][150/200] loss=0.576315 avg=0.654332 VRAM=38.6GiB | 18.8% done | ETA(epoch): 664s + [Epoch 93][160/200] loss=0.764078 avg=0.655579 VRAM=38.5GiB | 18.8% done | ETA(epoch): 532s + [Epoch 93][170/200] loss=0.683654 avg=0.654469 VRAM=38.6GiB | 18.8% done | ETA(epoch): 399s + [Epoch 93][180/200] loss=0.669684 avg=0.654559 VRAM=38.5GiB | 18.8% done | ETA(epoch): 266s + [Epoch 93][190/200] loss=0.814568 avg=0.655292 VRAM=38.6GiB | 18.8% done | ETA(epoch): 133s + [Epoch 93][200/200] loss=0.715462 avg=0.653704 VRAM=38.5GiB | 18.8% done | ETA(epoch): 0s + Train loss: 0.653704 (2657.6s) ETA: 19954min + Val loss: 0.683870 [t_0.0-0.2=0.9154 t_0.2-0.4=0.7427 t_0.4-0.6=0.6372 t_0.6-0.8=0.5636 t_0.8-1.0=0.5827] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 94/499 (19% done) --- + [Epoch 94][10/200] loss=0.666109 avg=0.666931 VRAM=38.6GiB | 18.8% done | ETA(epoch): 2516s + [Epoch 94][20/200] loss=0.530347 avg=0.654843 VRAM=38.5GiB | 18.8% done | ETA(epoch): 2387s + [Epoch 94][30/200] loss=0.779681 avg=0.659619 VRAM=38.6GiB | 18.8% done | ETA(epoch): 2254s + [Epoch 94][40/200] loss=0.712256 avg=0.671892 VRAM=38.5GiB | 18.8% done | ETA(epoch): 2122s + [Epoch 94][50/200] loss=0.713389 avg=0.671610 VRAM=38.6GiB | 18.9% done | ETA(epoch): 1989s + [Epoch 94][60/200] loss=0.701764 avg=0.672637 VRAM=38.5GiB | 18.9% done | ETA(epoch): 1857s + [Epoch 94][70/200] loss=0.634651 avg=0.672016 VRAM=38.6GiB | 18.9% done | ETA(epoch): 1724s + [Epoch 94][80/200] loss=0.837541 avg=0.672405 VRAM=38.5GiB | 18.9% done | ETA(epoch): 1592s + [Epoch 94][90/200] loss=0.685376 avg=0.671014 VRAM=38.6GiB | 18.9% done | ETA(epoch): 1459s + [Epoch 94][100/200] loss=0.722790 avg=0.666304 VRAM=38.5GiB | 18.9% done | ETA(epoch): 1327s + [Epoch 94][110/200] loss=0.662198 avg=0.664922 VRAM=38.6GiB | 18.9% done | ETA(epoch): 1194s + [Epoch 94][120/200] loss=0.654942 avg=0.662422 VRAM=38.5GiB | 18.9% done | ETA(epoch): 1061s + [Epoch 94][130/200] loss=0.627084 avg=0.661990 VRAM=38.6GiB | 18.9% done | ETA(epoch): 929s + [Epoch 94][140/200] loss=0.571805 avg=0.658025 VRAM=38.5GiB | 18.9% done | ETA(epoch): 796s + [Epoch 94][150/200] loss=0.670932 avg=0.661424 VRAM=38.6GiB | 18.9% done | ETA(epoch): 663s + [Epoch 94][160/200] loss=0.616727 avg=0.658178 VRAM=38.5GiB | 19.0% done | ETA(epoch): 531s + [Epoch 94][170/200] loss=0.613911 avg=0.658049 VRAM=38.6GiB | 19.0% done | ETA(epoch): 398s + [Epoch 94][180/200] loss=0.703301 avg=0.657747 VRAM=38.5GiB | 19.0% done | ETA(epoch): 265s + [Epoch 94][190/200] loss=0.892811 avg=0.658090 VRAM=38.6GiB | 19.0% done | ETA(epoch): 133s + [Epoch 94][200/200] loss=0.713879 avg=0.655970 VRAM=38.5GiB | 19.0% done | ETA(epoch): 0s + Train loss: 0.655970 (2654.1s) ETA: 19902min + Val loss: 0.681527 [t_0.0-0.2=0.9473 t_0.2-0.4=0.7636 t_0.4-0.6=0.6429 t_0.6-0.8=0.5602 t_0.8-1.0=0.5603] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 23.7/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 95/499 (19% done) --- + [Epoch 95][10/200] loss=0.692741 avg=0.683176 VRAM=38.6GiB | 19.0% done | ETA(epoch): 2570s + [Epoch 95][20/200] loss=0.645227 avg=0.663250 VRAM=38.5GiB | 19.0% done | ETA(epoch): 2411s + [Epoch 95][30/200] loss=0.687445 avg=0.656808 VRAM=38.6GiB | 19.0% done | ETA(epoch): 2270s + [Epoch 95][40/200] loss=0.600515 avg=0.650323 VRAM=38.5GiB | 19.0% done | ETA(epoch): 2134s + [Epoch 95][50/200] loss=0.670983 avg=0.646531 VRAM=38.6GiB | 19.1% done | ETA(epoch): 1998s + [Epoch 95][60/200] loss=0.656421 avg=0.645455 VRAM=38.5GiB | 19.1% done | ETA(epoch): 1864s + [Epoch 95][70/200] loss=0.784640 avg=0.646388 VRAM=38.6GiB | 19.1% done | ETA(epoch): 1729s + [Epoch 95][80/200] loss=0.614803 avg=0.649026 VRAM=38.5GiB | 19.1% done | ETA(epoch): 1596s + [Epoch 95][90/200] loss=0.685588 avg=0.652565 VRAM=38.6GiB | 19.1% done | ETA(epoch): 1462s + [Epoch 95][100/200] loss=0.567363 avg=0.652862 VRAM=38.5GiB | 19.1% done | ETA(epoch): 1329s + [Epoch 95][110/200] loss=0.671973 avg=0.652144 VRAM=38.6GiB | 19.1% done | ETA(epoch): 1196s + [Epoch 95][120/200] loss=0.749364 avg=0.654926 VRAM=38.5GiB | 19.1% done | ETA(epoch): 1063s + [Epoch 95][130/200] loss=0.565528 avg=0.651716 VRAM=38.6GiB | 19.1% done | ETA(epoch): 930s + [Epoch 95][140/200] loss=0.557934 avg=0.650864 VRAM=38.5GiB | 19.1% done | ETA(epoch): 797s + [Epoch 95][150/200] loss=0.816872 avg=0.653089 VRAM=38.6GiB | 19.1% done | ETA(epoch): 664s + [Epoch 95][160/200] loss=0.705687 avg=0.652435 VRAM=38.5GiB | 19.2% done | ETA(epoch): 531s + [Epoch 95][170/200] loss=0.706392 avg=0.653429 VRAM=38.6GiB | 19.2% done | ETA(epoch): 399s + [Epoch 95][180/200] loss=0.670387 avg=0.653782 VRAM=38.5GiB | 19.2% done | ETA(epoch): 266s + [Epoch 95][190/200] loss=0.938082 avg=0.655671 VRAM=38.6GiB | 19.2% done | ETA(epoch): 133s + [Epoch 95][200/200] loss=0.554358 avg=0.655427 VRAM=38.5GiB | 19.2% done | ETA(epoch): 0s + Train loss: 0.655427 (2659.2s) ETA: 19850min + Val loss: 0.696737 [t_0.0-0.2=0.9284 t_0.2-0.4=0.7487 t_0.4-0.6=0.6413 t_0.6-0.8=0.5643 t_0.8-1.0=0.5733] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 96/499 (19% done) --- + [Epoch 96][10/200] loss=0.786597 avg=0.680468 VRAM=38.6GiB | 19.2% done | ETA(epoch): 2522s + [Epoch 96][20/200] loss=0.595172 avg=0.660680 VRAM=38.5GiB | 19.2% done | ETA(epoch): 2390s + [Epoch 96][30/200] loss=0.601386 avg=0.656899 VRAM=38.6GiB | 19.2% done | ETA(epoch): 2256s + [Epoch 96][40/200] loss=0.557521 avg=0.652024 VRAM=38.5GiB | 19.2% done | ETA(epoch): 2124s + [Epoch 96][50/200] loss=0.712092 avg=0.654325 VRAM=38.6GiB | 19.2% done | ETA(epoch): 1990s + [Epoch 96][60/200] loss=0.605123 avg=0.654422 VRAM=38.5GiB | 19.3% done | ETA(epoch): 1858s + [Epoch 96][70/200] loss=0.626095 avg=0.655210 VRAM=38.6GiB | 19.3% done | ETA(epoch): 1725s + [Epoch 96][80/200] loss=0.611048 avg=0.658995 VRAM=38.5GiB | 19.3% done | ETA(epoch): 1592s + [Epoch 96][90/200] loss=0.613851 avg=0.661386 VRAM=38.6GiB | 19.3% done | ETA(epoch): 1459s + [Epoch 96][100/200] loss=0.573799 avg=0.661058 VRAM=38.5GiB | 19.3% done | ETA(epoch): 1327s + [Epoch 96][110/200] loss=0.585103 avg=0.662275 VRAM=38.6GiB | 19.3% done | ETA(epoch): 1194s + [Epoch 96][120/200] loss=0.586159 avg=0.667970 VRAM=38.5GiB | 19.3% done | ETA(epoch): 1061s + [Epoch 96][130/200] loss=0.691515 avg=0.667238 VRAM=38.6GiB | 19.3% done | ETA(epoch): 929s + [Epoch 96][140/200] loss=0.740061 avg=0.664004 VRAM=38.5GiB | 19.3% done | ETA(epoch): 796s + [Epoch 96][150/200] loss=0.706926 avg=0.666208 VRAM=38.6GiB | 19.4% done | ETA(epoch): 664s + [Epoch 96][160/200] loss=0.591956 avg=0.663820 VRAM=38.5GiB | 19.4% done | ETA(epoch): 531s + [Epoch 96][170/200] loss=0.535335 avg=0.663421 VRAM=38.6GiB | 19.4% done | ETA(epoch): 398s + [Epoch 96][180/200] loss=0.619281 avg=0.664285 VRAM=38.5GiB | 19.4% done | ETA(epoch): 265s + [Epoch 96][190/200] loss=0.558176 avg=0.663262 VRAM=38.6GiB | 19.4% done | ETA(epoch): 133s + [Epoch 96][200/200] loss=0.761978 avg=0.662900 VRAM=38.5GiB | 19.4% done | ETA(epoch): 0s + Train loss: 0.662900 (2653.8s) ETA: 19798min + Val loss: 0.697854 [t_0.0-0.2=0.9296 t_0.2-0.4=0.7556 t_0.4-0.6=0.6425 t_0.6-0.8=0.5594 t_0.8-1.0=0.5759] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 97/499 (19% done) --- + [Epoch 97][10/200] loss=0.690687 avg=0.674696 VRAM=38.6GiB | 19.4% done | ETA(epoch): 2522s + [Epoch 97][20/200] loss=0.732651 avg=0.674554 VRAM=38.5GiB | 19.4% done | ETA(epoch): 2411s + [Epoch 97][30/200] loss=0.647643 avg=0.662970 VRAM=38.6GiB | 19.4% done | ETA(epoch): 2270s + [Epoch 97][40/200] loss=0.584481 avg=0.664616 VRAM=38.5GiB | 19.4% done | ETA(epoch): 2134s + [Epoch 97][50/200] loss=0.648965 avg=0.665117 VRAM=38.6GiB | 19.4% done | ETA(epoch): 1998s + [Epoch 97][60/200] loss=0.583222 avg=0.662460 VRAM=38.5GiB | 19.5% done | ETA(epoch): 1865s + [Epoch 97][70/200] loss=0.621119 avg=0.660322 VRAM=38.6GiB | 19.5% done | ETA(epoch): 1730s + [Epoch 97][80/200] loss=0.627143 avg=0.660126 VRAM=38.5GiB | 19.5% done | ETA(epoch): 1597s + [Epoch 97][90/200] loss=0.612365 avg=0.655563 VRAM=38.6GiB | 19.5% done | ETA(epoch): 1463s + [Epoch 97][100/200] loss=0.734045 avg=0.656573 VRAM=38.5GiB | 19.5% done | ETA(epoch): 1330s + [Epoch 97][110/200] loss=0.631944 avg=0.659931 VRAM=38.6GiB | 19.5% done | ETA(epoch): 1197s + [Epoch 97][120/200] loss=0.711509 avg=0.661697 VRAM=38.5GiB | 19.5% done | ETA(epoch): 1064s + [Epoch 97][130/200] loss=0.610898 avg=0.661968 VRAM=38.6GiB | 19.5% done | ETA(epoch): 931s + [Epoch 97][140/200] loss=0.699439 avg=0.661498 VRAM=38.5GiB | 19.5% done | ETA(epoch): 798s + [Epoch 97][150/200] loss=0.702193 avg=0.662392 VRAM=38.6GiB | 19.6% done | ETA(epoch): 665s + [Epoch 97][160/200] loss=0.657343 avg=0.662290 VRAM=38.5GiB | 19.6% done | ETA(epoch): 532s + [Epoch 97][170/200] loss=0.631945 avg=0.662412 VRAM=38.6GiB | 19.6% done | ETA(epoch): 399s + [Epoch 97][180/200] loss=0.734823 avg=0.662687 VRAM=38.5GiB | 19.6% done | ETA(epoch): 266s + [Epoch 97][190/200] loss=0.686339 avg=0.662469 VRAM=38.6GiB | 19.6% done | ETA(epoch): 133s + [Epoch 97][200/200] loss=0.914138 avg=0.662908 VRAM=38.5GiB | 19.6% done | ETA(epoch): 0s + Train loss: 0.662908 (2657.5s) ETA: 19746min + Val loss: 0.674025 [t_0.0-0.2=0.9263 t_0.2-0.4=0.7482 t_0.4-0.6=0.6063 t_0.6-0.8=0.5655 t_0.8-1.0=0.5756] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0097 (BEST) + Deleted old checkpoint: checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 98/499 (20% done) --- + [Epoch 98][10/200] loss=0.673457 avg=0.675323 VRAM=38.6GiB | 19.6% done | ETA(epoch): 2520s + [Epoch 98][20/200] loss=0.579789 avg=0.659235 VRAM=38.5GiB | 19.6% done | ETA(epoch): 2388s + [Epoch 98][30/200] loss=0.695864 avg=0.662108 VRAM=38.6GiB | 19.6% done | ETA(epoch): 2257s + [Epoch 98][40/200] loss=0.554833 avg=0.663231 VRAM=38.5GiB | 19.6% done | ETA(epoch): 2123s + [Epoch 98][50/200] loss=0.681481 avg=0.667137 VRAM=38.6GiB | 19.7% done | ETA(epoch): 1990s + [Epoch 98][60/200] loss=0.734608 avg=0.660346 VRAM=38.5GiB | 19.7% done | ETA(epoch): 1858s + [Epoch 98][70/200] loss=0.673610 avg=0.659408 VRAM=38.6GiB | 19.7% done | ETA(epoch): 1725s + [Epoch 98][80/200] loss=0.663754 avg=0.662431 VRAM=38.5GiB | 19.7% done | ETA(epoch): 1596s + [Epoch 98][90/200] loss=0.726041 avg=0.662054 VRAM=38.6GiB | 19.7% done | ETA(epoch): 1463s + [Epoch 98][100/200] loss=0.666424 avg=0.661936 VRAM=38.5GiB | 19.7% done | ETA(epoch): 1330s + [Epoch 98][110/200] loss=0.694103 avg=0.659008 VRAM=38.6GiB | 19.7% done | ETA(epoch): 1196s + [Epoch 98][120/200] loss=0.646462 avg=0.661725 VRAM=38.5GiB | 19.7% done | ETA(epoch): 1064s + [Epoch 98][130/200] loss=0.693820 avg=0.661080 VRAM=38.6GiB | 19.7% done | ETA(epoch): 930s + [Epoch 98][140/200] loss=0.731449 avg=0.659565 VRAM=38.5GiB | 19.7% done | ETA(epoch): 798s + [Epoch 98][150/200] loss=0.669937 avg=0.663802 VRAM=38.6GiB | 19.8% done | ETA(epoch): 665s + [Epoch 98][160/200] loss=0.597537 avg=0.664220 VRAM=38.5GiB | 19.8% done | ETA(epoch): 532s + [Epoch 98][170/200] loss=0.774962 avg=0.666132 VRAM=38.6GiB | 19.8% done | ETA(epoch): 399s + [Epoch 98][180/200] loss=0.650587 avg=0.665391 VRAM=38.5GiB | 19.8% done | ETA(epoch): 266s + [Epoch 98][190/200] loss=0.830103 avg=0.664899 VRAM=38.6GiB | 19.8% done | ETA(epoch): 133s + [Epoch 98][200/200] loss=0.646179 avg=0.665185 VRAM=38.5GiB | 19.8% done | ETA(epoch): 0s + Train loss: 0.665185 (2660.5s) ETA: 19696min + Val loss: 0.688429 [t_0.0-0.2=0.9270 t_0.2-0.4=0.7431 t_0.4-0.6=0.6440 t_0.6-0.8=0.5688 t_0.8-1.0=0.5634] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 99/499 (20% done) --- + [Epoch 99][10/200] loss=0.654904 avg=0.642150 VRAM=38.6GiB | 19.8% done | ETA(epoch): 2519s + [Epoch 99][20/200] loss=0.540991 avg=0.641839 VRAM=38.5GiB | 19.8% done | ETA(epoch): 2391s + [Epoch 99][30/200] loss=0.767655 avg=0.651638 VRAM=38.6GiB | 19.8% done | ETA(epoch): 2256s + [Epoch 99][40/200] loss=0.585631 avg=0.655680 VRAM=38.5GiB | 19.8% done | ETA(epoch): 2125s + [Epoch 99][50/200] loss=0.752582 avg=0.654838 VRAM=38.6GiB | 19.9% done | ETA(epoch): 1992s + [Epoch 99][60/200] loss=0.782093 avg=0.661820 VRAM=38.5GiB | 19.9% done | ETA(epoch): 1860s + [Epoch 99][70/200] loss=0.766895 avg=0.666849 VRAM=38.6GiB | 19.9% done | ETA(epoch): 1726s + [Epoch 99][80/200] loss=0.589004 avg=0.661981 VRAM=38.5GiB | 19.9% done | ETA(epoch): 1594s + [Epoch 99][90/200] loss=0.685740 avg=0.660532 VRAM=38.6GiB | 19.9% done | ETA(epoch): 1461s + [Epoch 99][100/200] loss=0.673551 avg=0.659277 VRAM=38.5GiB | 19.9% done | ETA(epoch): 1328s + [Epoch 99][110/200] loss=0.603816 avg=0.660358 VRAM=38.6GiB | 19.9% done | ETA(epoch): 1195s + [Epoch 99][120/200] loss=0.706123 avg=0.662506 VRAM=38.5GiB | 19.9% done | ETA(epoch): 1062s + [Epoch 99][130/200] loss=0.642577 avg=0.661674 VRAM=38.6GiB | 19.9% done | ETA(epoch): 929s + [Epoch 99][140/200] loss=0.722263 avg=0.661970 VRAM=38.5GiB | 19.9% done | ETA(epoch): 797s + [Epoch 99][150/200] loss=0.684207 avg=0.660545 VRAM=38.6GiB | 20.0% done | ETA(epoch): 664s + [Epoch 99][160/200] loss=0.653288 avg=0.661148 VRAM=38.5GiB | 20.0% done | ETA(epoch): 531s + [Epoch 99][170/200] loss=0.656135 avg=0.662006 VRAM=38.6GiB | 20.0% done | ETA(epoch): 398s + [Epoch 99][180/200] loss=0.853559 avg=0.663122 VRAM=38.5GiB | 20.0% done | ETA(epoch): 266s + [Epoch 99][190/200] loss=0.777591 avg=0.662145 VRAM=38.6GiB | 20.0% done | ETA(epoch): 133s + [Epoch 99][200/200] loss=0.671073 avg=0.663617 VRAM=38.5GiB | 20.0% done | ETA(epoch): 0s + Train loss: 0.663617 (2654.8s) ETA: 19644min + Val loss: 0.688423 [t_0.0-0.2=0.9471 t_0.2-0.4=0.7406 t_0.4-0.6=0.6271 t_0.6-0.8=0.5610 t_0.8-1.0=0.5794] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 100/499 (20% done) --- + [MilestoneVis] train_0 step 20000 ✓ + [MilestoneGrid] train_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 20000 ✓ + [MilestoneGrid] train_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 20000 ✓ + [MilestoneGrid] val_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 20000 ✓ + [MilestoneGrid] val_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 20000 ✓ + [MilestoneGrid] test_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 20000 ✓ + [MilestoneGrid] test_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 20000 ✓ + [MilestoneGrid] test_2 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 20000 ✓ + [MilestoneGrid] test_3 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 20000 ✓ + [MilestoneGrid] test_4 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 20000 ✓ + [MilestoneGrid] test_5 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_5.png + [MilestoneVis] test_6 step 20000 ✓ + [MilestoneGrid] test_6 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_6.png + [MilestoneVis] test_7 step 20000 ✓ + [MilestoneGrid] test_7 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_7.png + [MilestoneVis] test_8 step 20000 ✓ + [MilestoneGrid] test_8 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_8.png + [MilestoneVis] test_9 step 20000 ✓ + [MilestoneGrid] test_9 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_9.png + [MilestoneVis] test_10 step 20000 ✓ + [MilestoneGrid] test_10 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_10.png + [MilestoneVis] test_11 step 20000 ✓ + [MilestoneGrid] test_11 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_11.png + [MilestoneVis] test_12 step 20000 ✓ + [MilestoneGrid] test_12 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_12.png + [MilestoneVis] test_13 step 20000 ✓ + [MilestoneGrid] test_13 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_13.png + [MilestoneVis] test_14 step 20000 ✓ + [MilestoneGrid] test_14 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_14.png + [MilestoneVis] test_15 step 20000 ✓ + [MilestoneGrid] test_15 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_15.png + [MilestoneVis] test_16 step 20000 ✓ + [MilestoneGrid] test_16 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_16.png + [Epoch 100][10/200] loss=0.700206 avg=0.658639 VRAM=38.6GiB | 20.0% done | ETA(epoch): 43853s + [Epoch 100][20/200] loss=0.593843 avg=0.663786 VRAM=38.5GiB | 20.0% done | ETA(epoch): 21967s + [Epoch 100][30/200] loss=0.649298 avg=0.659113 VRAM=38.6GiB | 20.0% done | ETA(epoch): 14583s + [Epoch 100][40/200] loss=0.683333 avg=0.658458 VRAM=38.5GiB | 20.0% done | ETA(epoch): 10825s + [Epoch 100][50/200] loss=0.664660 avg=0.655962 VRAM=38.6GiB | 20.1% done | ETA(epoch): 8517s + [Epoch 100][60/200] loss=0.654520 avg=0.658391 VRAM=38.5GiB | 20.1% done | ETA(epoch): 6934s + [Epoch 100][70/200] loss=0.694708 avg=0.661415 VRAM=38.6GiB | 20.1% done | ETA(epoch): 5765s + [Epoch 100][80/200] loss=0.694668 avg=0.659626 VRAM=38.5GiB | 20.1% done | ETA(epoch): 4856s + [Epoch 100][90/200] loss=0.658192 avg=0.658561 VRAM=38.6GiB | 20.1% done | ETA(epoch): 4119s + [Epoch 100][100/200] loss=0.632946 avg=0.661179 VRAM=38.5GiB | 20.1% done | ETA(epoch): 3503s + [Epoch 100][110/200] loss=0.634742 avg=0.655695 VRAM=38.6GiB | 20.1% done | ETA(epoch): 2974s + [Epoch 100][120/200] loss=0.833635 avg=0.655734 VRAM=38.5GiB | 20.1% done | ETA(epoch): 2512s + [Epoch 100][130/200] loss=0.896838 avg=0.657271 VRAM=38.6GiB | 20.1% done | ETA(epoch): 2100s + [Epoch 100][140/200] loss=0.597395 avg=0.658882 VRAM=38.5GiB | 20.1% done | ETA(epoch): 1728s + [Epoch 100][150/200] loss=0.587468 avg=0.658048 VRAM=38.6GiB | 20.2% done | ETA(epoch): 1389s + [Epoch 100][160/200] loss=0.560917 avg=0.657944 VRAM=38.5GiB | 20.2% done | ETA(epoch): 1075s + [Epoch 100][170/200] loss=0.623049 avg=0.658500 VRAM=38.6GiB | 20.2% done | ETA(epoch): 782s + [Epoch 100][180/200] loss=0.725502 avg=0.657099 VRAM=38.5GiB | 20.2% done | ETA(epoch): 507s + [Epoch 100][190/200] loss=0.662703 avg=0.658607 VRAM=38.6GiB | 20.2% done | ETA(epoch): 247s + [Epoch 100][200/200] loss=0.718844 avg=0.658282 VRAM=38.5GiB | 20.2% done | ETA(epoch): 0s + Train loss: 0.658282 (4829.9s) ETA: 19735min + Val loss: 0.697207 [t_0.0-0.2=0.9254 t_0.2-0.4=0.7571 t_0.4-0.6=0.6457 t_0.6-0.8=0.5438 t_0.8-1.0=0.5775] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0100 +[MEM @ epoch 100 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 101/499 (20% done) --- + [Epoch 101][10/200] loss=0.609188 avg=0.662337 VRAM=38.6GiB | 20.2% done | ETA(epoch): 2519s + [Epoch 101][20/200] loss=0.823234 avg=0.656818 VRAM=38.5GiB | 20.2% done | ETA(epoch): 2386s + [Epoch 101][30/200] loss=0.711188 avg=0.662238 VRAM=38.6GiB | 20.2% done | ETA(epoch): 2256s + [Epoch 101][40/200] loss=0.554407 avg=0.663289 VRAM=38.5GiB | 20.2% done | ETA(epoch): 2123s + [Epoch 101][50/200] loss=0.708911 avg=0.666097 VRAM=38.6GiB | 20.2% done | ETA(epoch): 1991s + [Epoch 101][60/200] loss=0.564943 avg=0.664914 VRAM=38.5GiB | 20.3% done | ETA(epoch): 1858s + [Epoch 101][70/200] loss=0.596594 avg=0.659396 VRAM=38.6GiB | 20.3% done | ETA(epoch): 1726s + [Epoch 101][80/200] loss=0.560867 avg=0.656328 VRAM=38.5GiB | 20.3% done | ETA(epoch): 1593s + [Epoch 101][90/200] loss=0.666692 avg=0.654709 VRAM=38.6GiB | 20.3% done | ETA(epoch): 1460s + [Epoch 101][100/200] loss=0.790108 avg=0.660030 VRAM=38.5GiB | 20.3% done | ETA(epoch): 1327s + [Epoch 101][110/200] loss=0.721467 avg=0.658052 VRAM=38.6GiB | 20.3% done | ETA(epoch): 1194s + [Epoch 101][120/200] loss=0.688855 avg=0.659491 VRAM=38.5GiB | 20.3% done | ETA(epoch): 1062s + [Epoch 101][130/200] loss=0.654973 avg=0.658731 VRAM=38.6GiB | 20.3% done | ETA(epoch): 929s + [Epoch 101][140/200] loss=0.656953 avg=0.659856 VRAM=38.5GiB | 20.3% done | ETA(epoch): 796s + [Epoch 101][150/200] loss=0.608622 avg=0.660486 VRAM=38.6GiB | 20.3% done | ETA(epoch): 664s + [Epoch 101][160/200] loss=0.635207 avg=0.662810 VRAM=38.5GiB | 20.4% done | ETA(epoch): 531s + [Epoch 101][170/200] loss=0.721342 avg=0.662290 VRAM=38.6GiB | 20.4% done | ETA(epoch): 398s + [Epoch 101][180/200] loss=0.727218 avg=0.664627 VRAM=38.5GiB | 20.4% done | ETA(epoch): 265s + [Epoch 101][190/200] loss=0.665687 avg=0.663233 VRAM=38.6GiB | 20.4% done | ETA(epoch): 133s + [Epoch 101][200/200] loss=0.611508 avg=0.664094 VRAM=38.5GiB | 20.4% done | ETA(epoch): 0s + Train loss: 0.664094 (2654.0s) ETA: 19681min + Val loss: 0.694533 [t_0.0-0.2=0.9185 t_0.2-0.4=0.7527 t_0.4-0.6=0.6445 t_0.6-0.8=0.5655 t_0.8-1.0=0.5851] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 102/499 (20% done) --- + [Epoch 102][10/200] loss=0.777205 avg=0.651973 VRAM=38.6GiB | 20.4% done | ETA(epoch): 2521s + [Epoch 102][20/200] loss=0.540417 avg=0.665660 VRAM=38.5GiB | 20.4% done | ETA(epoch): 2388s + [Epoch 102][30/200] loss=0.590842 avg=0.654692 VRAM=38.6GiB | 20.4% done | ETA(epoch): 2257s + [Epoch 102][40/200] loss=0.496740 avg=0.649236 VRAM=38.5GiB | 20.4% done | ETA(epoch): 2124s + [Epoch 102][50/200] loss=0.639331 avg=0.644929 VRAM=38.6GiB | 20.4% done | ETA(epoch): 1992s + [Epoch 102][60/200] loss=0.814636 avg=0.646002 VRAM=38.5GiB | 20.5% done | ETA(epoch): 1859s + [Epoch 102][70/200] loss=0.579116 avg=0.644897 VRAM=38.6GiB | 20.5% done | ETA(epoch): 1726s + [Epoch 102][80/200] loss=0.690288 avg=0.645920 VRAM=38.5GiB | 20.5% done | ETA(epoch): 1593s + [Epoch 102][90/200] loss=0.642491 avg=0.647401 VRAM=38.6GiB | 20.5% done | ETA(epoch): 1460s + [Epoch 102][100/200] loss=0.678084 avg=0.652096 VRAM=38.5GiB | 20.5% done | ETA(epoch): 1328s + [Epoch 102][110/200] loss=0.757910 avg=0.651738 VRAM=38.6GiB | 20.5% done | ETA(epoch): 1195s + [Epoch 102][120/200] loss=0.546611 avg=0.647966 VRAM=38.5GiB | 20.5% done | ETA(epoch): 1062s + [Epoch 102][130/200] loss=0.671356 avg=0.648667 VRAM=38.6GiB | 20.5% done | ETA(epoch): 929s + [Epoch 102][140/200] loss=0.586569 avg=0.650389 VRAM=38.5GiB | 20.5% done | ETA(epoch): 797s + [Epoch 102][150/200] loss=0.597603 avg=0.649819 VRAM=38.6GiB | 20.5% done | ETA(epoch): 664s + [Epoch 102][160/200] loss=0.568209 avg=0.649023 VRAM=38.5GiB | 20.6% done | ETA(epoch): 531s + [Epoch 102][170/200] loss=0.666555 avg=0.648701 VRAM=38.6GiB | 20.6% done | ETA(epoch): 398s + [Epoch 102][180/200] loss=0.621565 avg=0.650569 VRAM=38.5GiB | 20.6% done | ETA(epoch): 265s + [Epoch 102][190/200] loss=0.811550 avg=0.651081 VRAM=38.6GiB | 20.6% done | ETA(epoch): 133s + [Epoch 102][200/200] loss=0.628573 avg=0.651024 VRAM=38.5GiB | 20.6% done | ETA(epoch): 0s + Train loss: 0.651024 (2654.6s) ETA: 19628min + Val loss: 0.693420 [t_0.0-0.2=0.9056 t_0.2-0.4=0.7528 t_0.4-0.6=0.6510 t_0.6-0.8=0.5686 t_0.8-1.0=0.5734] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 103/499 (21% done) --- + [Epoch 103][10/200] loss=0.734090 avg=0.686050 VRAM=38.6GiB | 20.6% done | ETA(epoch): 2523s + [Epoch 103][20/200] loss=0.587606 avg=0.680546 VRAM=38.5GiB | 20.6% done | ETA(epoch): 2388s + [Epoch 103][30/200] loss=0.791769 avg=0.668906 VRAM=38.6GiB | 20.6% done | ETA(epoch): 2256s + [Epoch 103][40/200] loss=0.632967 avg=0.659476 VRAM=38.5GiB | 20.6% done | ETA(epoch): 2123s + [Epoch 103][50/200] loss=0.616954 avg=0.663018 VRAM=38.6GiB | 20.6% done | ETA(epoch): 1990s + [Epoch 103][60/200] loss=0.701427 avg=0.664617 VRAM=38.5GiB | 20.7% done | ETA(epoch): 1857s + [Epoch 103][70/200] loss=0.585365 avg=0.673806 VRAM=38.6GiB | 20.7% done | ETA(epoch): 1725s + [Epoch 103][80/200] loss=0.638847 avg=0.672937 VRAM=38.5GiB | 20.7% done | ETA(epoch): 1592s + [Epoch 103][90/200] loss=0.620574 avg=0.673069 VRAM=38.6GiB | 20.7% done | ETA(epoch): 1462s + [Epoch 103][100/200] loss=0.557441 avg=0.670800 VRAM=38.5GiB | 20.7% done | ETA(epoch): 1329s + [Epoch 103][110/200] loss=0.663322 avg=0.667722 VRAM=38.6GiB | 20.7% done | ETA(epoch): 1196s + [Epoch 103][120/200] loss=0.686192 avg=0.666555 VRAM=38.5GiB | 20.7% done | ETA(epoch): 1063s + [Epoch 103][130/200] loss=0.552635 avg=0.665423 VRAM=38.6GiB | 20.7% done | ETA(epoch): 930s + [Epoch 103][140/200] loss=0.654690 avg=0.662110 VRAM=38.5GiB | 20.7% done | ETA(epoch): 797s + [Epoch 103][150/200] loss=0.569706 avg=0.661105 VRAM=38.6GiB | 20.8% done | ETA(epoch): 664s + [Epoch 103][160/200] loss=0.713046 avg=0.661267 VRAM=38.5GiB | 20.8% done | ETA(epoch): 531s + [Epoch 103][170/200] loss=0.678152 avg=0.662818 VRAM=38.6GiB | 20.8% done | ETA(epoch): 398s + [Epoch 103][180/200] loss=0.674646 avg=0.664408 VRAM=38.5GiB | 20.8% done | ETA(epoch): 266s + [Epoch 103][190/200] loss=0.638858 avg=0.663486 VRAM=38.6GiB | 20.8% done | ETA(epoch): 133s + [Epoch 103][200/200] loss=0.784176 avg=0.663521 VRAM=38.5GiB | 20.8% done | ETA(epoch): 0s + Train loss: 0.663521 (2656.0s) ETA: 19575min + Val loss: 0.709770 [t_0.0-0.2=0.9433 t_0.2-0.4=0.7495 t_0.4-0.6=0.6452 t_0.6-0.8=0.5588 t_0.8-1.0=0.5907] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 25.5/188.5 GiB (13.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 104/499 (21% done) --- + [Epoch 104][10/200] loss=0.675117 avg=0.661253 VRAM=38.6GiB | 20.8% done | ETA(epoch): 2521s + [Epoch 104][20/200] loss=0.677293 avg=0.642046 VRAM=38.5GiB | 20.8% done | ETA(epoch): 2387s + [Epoch 104][30/200] loss=0.602575 avg=0.649884 VRAM=38.6GiB | 20.8% done | ETA(epoch): 2255s + [Epoch 104][40/200] loss=0.681854 avg=0.651783 VRAM=38.5GiB | 20.8% done | ETA(epoch): 2122s + [Epoch 104][50/200] loss=0.660064 avg=0.653234 VRAM=38.6GiB | 20.8% done | ETA(epoch): 1990s + [Epoch 104][60/200] loss=0.707983 avg=0.656738 VRAM=38.5GiB | 20.9% done | ETA(epoch): 1857s + [Epoch 104][70/200] loss=0.631555 avg=0.658771 VRAM=38.6GiB | 20.9% done | ETA(epoch): 1725s + [Epoch 104][80/200] loss=0.736635 avg=0.654438 VRAM=38.5GiB | 20.9% done | ETA(epoch): 1592s + [Epoch 104][90/200] loss=0.714761 avg=0.657877 VRAM=38.6GiB | 20.9% done | ETA(epoch): 1460s + [Epoch 104][100/200] loss=0.635905 avg=0.660561 VRAM=38.5GiB | 20.9% done | ETA(epoch): 1327s + [Epoch 104][110/200] loss=0.677642 avg=0.661599 VRAM=38.6GiB | 20.9% done | ETA(epoch): 1195s + [Epoch 104][120/200] loss=0.740353 avg=0.663885 VRAM=38.5GiB | 20.9% done | ETA(epoch): 1062s + [Epoch 104][130/200] loss=0.653438 avg=0.663444 VRAM=38.6GiB | 20.9% done | ETA(epoch): 929s + [Epoch 104][140/200] loss=0.543095 avg=0.663115 VRAM=38.5GiB | 20.9% done | ETA(epoch): 798s + [Epoch 104][150/200] loss=0.698859 avg=0.664011 VRAM=38.6GiB | 20.9% done | ETA(epoch): 665s + [Epoch 104][160/200] loss=0.630657 avg=0.663122 VRAM=38.5GiB | 21.0% done | ETA(epoch): 532s + [Epoch 104][170/200] loss=0.570348 avg=0.663580 VRAM=38.6GiB | 21.0% done | ETA(epoch): 399s + [Epoch 104][180/200] loss=0.823470 avg=0.662544 VRAM=38.5GiB | 21.0% done | ETA(epoch): 266s + [Epoch 104][190/200] loss=0.698210 avg=0.663362 VRAM=38.6GiB | 21.0% done | ETA(epoch): 133s + [Epoch 104][200/200] loss=0.632063 avg=0.661918 VRAM=38.5GiB | 21.0% done | ETA(epoch): 0s + Train loss: 0.661918 (2657.9s) ETA: 19522min + Val loss: 0.702384 [t_0.0-0.2=0.9458 t_0.2-0.4=0.7586 t_0.4-0.6=0.6389 t_0.6-0.8=0.5589 t_0.8-1.0=0.5595] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 105/499 (21% done) --- + [Epoch 105][10/200] loss=0.608074 avg=0.691476 VRAM=38.6GiB | 21.0% done | ETA(epoch): 2576s + [Epoch 105][20/200] loss=0.780031 avg=0.664528 VRAM=38.5GiB | 21.0% done | ETA(epoch): 2415s + [Epoch 105][30/200] loss=0.698001 avg=0.660397 VRAM=38.6GiB | 21.0% done | ETA(epoch): 2272s + [Epoch 105][40/200] loss=0.669007 avg=0.654656 VRAM=38.5GiB | 21.0% done | ETA(epoch): 2136s + [Epoch 105][50/200] loss=0.699644 avg=0.664681 VRAM=38.6GiB | 21.1% done | ETA(epoch): 1999s + [Epoch 105][60/200] loss=0.546226 avg=0.663135 VRAM=38.5GiB | 21.1% done | ETA(epoch): 1865s + [Epoch 105][70/200] loss=0.581601 avg=0.666104 VRAM=38.6GiB | 21.1% done | ETA(epoch): 1730s + [Epoch 105][80/200] loss=0.619298 avg=0.660560 VRAM=38.5GiB | 21.1% done | ETA(epoch): 1597s + [Epoch 105][90/200] loss=0.588232 avg=0.656311 VRAM=38.6GiB | 21.1% done | ETA(epoch): 1463s + [Epoch 105][100/200] loss=0.589666 avg=0.654921 VRAM=38.5GiB | 21.1% done | ETA(epoch): 1330s + [Epoch 105][110/200] loss=0.788981 avg=0.659267 VRAM=38.6GiB | 21.1% done | ETA(epoch): 1197s + [Epoch 105][120/200] loss=0.578656 avg=0.655093 VRAM=38.5GiB | 21.1% done | ETA(epoch): 1064s + [Epoch 105][130/200] loss=0.572167 avg=0.653212 VRAM=38.6GiB | 21.1% done | ETA(epoch): 930s + [Epoch 105][140/200] loss=0.780279 avg=0.650764 VRAM=38.5GiB | 21.1% done | ETA(epoch): 798s + [Epoch 105][150/200] loss=0.649446 avg=0.649516 VRAM=38.6GiB | 21.1% done | ETA(epoch): 664s + [Epoch 105][160/200] loss=0.697101 avg=0.652920 VRAM=38.5GiB | 21.2% done | ETA(epoch): 532s + [Epoch 105][170/200] loss=0.646006 avg=0.653280 VRAM=38.6GiB | 21.2% done | ETA(epoch): 399s + [Epoch 105][180/200] loss=0.712402 avg=0.652607 VRAM=38.5GiB | 21.2% done | ETA(epoch): 266s + [Epoch 105][190/200] loss=0.642086 avg=0.653178 VRAM=38.6GiB | 21.2% done | ETA(epoch): 133s + [Epoch 105][200/200] loss=0.724538 avg=0.654802 VRAM=38.5GiB | 21.2% done | ETA(epoch): 0s + Train loss: 0.654802 (2660.3s) ETA: 19469min + Val loss: 0.698006 [t_0.0-0.2=0.9711 t_0.2-0.4=0.7602 t_0.4-0.6=0.6350 t_0.6-0.8=0.5817 t_0.8-1.0=0.5656] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 106/499 (21% done) --- + [Epoch 106][10/200] loss=0.686279 avg=0.620038 VRAM=38.6GiB | 21.2% done | ETA(epoch): 2520s + [Epoch 106][20/200] loss=0.814618 avg=0.675502 VRAM=38.5GiB | 21.2% done | ETA(epoch): 2389s + [Epoch 106][30/200] loss=0.685730 avg=0.671559 VRAM=38.6GiB | 21.2% done | ETA(epoch): 2258s + [Epoch 106][40/200] loss=0.695361 avg=0.674543 VRAM=38.5GiB | 21.2% done | ETA(epoch): 2126s + [Epoch 106][50/200] loss=0.662349 avg=0.666566 VRAM=38.6GiB | 21.2% done | ETA(epoch): 1992s + [Epoch 106][60/200] loss=0.822315 avg=0.670575 VRAM=38.5GiB | 21.3% done | ETA(epoch): 1860s + [Epoch 106][70/200] loss=0.707392 avg=0.665724 VRAM=38.6GiB | 21.3% done | ETA(epoch): 1726s + [Epoch 106][80/200] loss=0.657682 avg=0.665096 VRAM=38.5GiB | 21.3% done | ETA(epoch): 1594s + [Epoch 106][90/200] loss=0.630909 avg=0.663850 VRAM=38.6GiB | 21.3% done | ETA(epoch): 1461s + [Epoch 106][100/200] loss=0.703547 avg=0.661424 VRAM=38.5GiB | 21.3% done | ETA(epoch): 1328s + [Epoch 106][110/200] loss=0.653568 avg=0.661604 VRAM=38.6GiB | 21.3% done | ETA(epoch): 1195s + [Epoch 106][120/200] loss=0.609946 avg=0.662383 VRAM=38.5GiB | 21.3% done | ETA(epoch): 1062s + [Epoch 106][130/200] loss=0.594273 avg=0.662941 VRAM=38.6GiB | 21.3% done | ETA(epoch): 930s + [Epoch 106][140/200] loss=0.716238 avg=0.662240 VRAM=38.5GiB | 21.3% done | ETA(epoch): 797s + [Epoch 106][150/200] loss=0.631364 avg=0.661302 VRAM=38.6GiB | 21.3% done | ETA(epoch): 664s + [Epoch 106][160/200] loss=0.771309 avg=0.660521 VRAM=38.5GiB | 21.4% done | ETA(epoch): 532s + [Epoch 106][170/200] loss=0.706421 avg=0.662679 VRAM=38.6GiB | 21.4% done | ETA(epoch): 399s + [Epoch 106][180/200] loss=0.679410 avg=0.663386 VRAM=38.5GiB | 21.4% done | ETA(epoch): 266s + [Epoch 106][190/200] loss=0.616277 avg=0.660925 VRAM=38.6GiB | 21.4% done | ETA(epoch): 133s + [Epoch 106][200/200] loss=0.641855 avg=0.660963 VRAM=38.5GiB | 21.4% done | ETA(epoch): 0s + Train loss: 0.660963 (2658.3s) ETA: 19416min + Val loss: 0.686422 [t_0.0-0.2=0.9332 t_0.2-0.4=0.7439 t_0.4-0.6=0.6370 t_0.6-0.8=0.5706 t_0.8-1.0=0.5702] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 107/499 (21% done) --- + [Epoch 107][10/200] loss=0.814270 avg=0.674327 VRAM=38.6GiB | 21.4% done | ETA(epoch): 2518s + [Epoch 107][20/200] loss=0.683063 avg=0.663523 VRAM=38.5GiB | 21.4% done | ETA(epoch): 2386s + [Epoch 107][30/200] loss=0.641643 avg=0.665515 VRAM=38.6GiB | 21.4% done | ETA(epoch): 2270s + [Epoch 107][40/200] loss=0.758014 avg=0.659111 VRAM=38.5GiB | 21.4% done | ETA(epoch): 2133s + [Epoch 107][50/200] loss=0.499890 avg=0.651749 VRAM=38.6GiB | 21.4% done | ETA(epoch): 1998s + [Epoch 107][60/200] loss=0.564135 avg=0.647459 VRAM=38.5GiB | 21.5% done | ETA(epoch): 1863s + [Epoch 107][70/200] loss=0.693492 avg=0.654947 VRAM=38.6GiB | 21.5% done | ETA(epoch): 1730s + [Epoch 107][80/200] loss=0.726566 avg=0.656311 VRAM=38.5GiB | 21.5% done | ETA(epoch): 1596s + [Epoch 107][90/200] loss=0.734068 avg=0.660355 VRAM=38.6GiB | 21.5% done | ETA(epoch): 1463s + [Epoch 107][100/200] loss=0.810194 avg=0.660064 VRAM=38.5GiB | 21.5% done | ETA(epoch): 1329s + [Epoch 107][110/200] loss=0.504903 avg=0.660755 VRAM=38.6GiB | 21.5% done | ETA(epoch): 1196s + [Epoch 107][120/200] loss=0.621951 avg=0.660488 VRAM=38.5GiB | 21.5% done | ETA(epoch): 1065s + [Epoch 107][130/200] loss=0.669286 avg=0.661477 VRAM=38.6GiB | 21.5% done | ETA(epoch): 932s + [Epoch 107][140/200] loss=0.602145 avg=0.659224 VRAM=38.5GiB | 21.5% done | ETA(epoch): 799s + [Epoch 107][150/200] loss=0.583067 avg=0.658395 VRAM=38.6GiB | 21.6% done | ETA(epoch): 665s + [Epoch 107][160/200] loss=0.582702 avg=0.659035 VRAM=38.5GiB | 21.6% done | ETA(epoch): 532s + [Epoch 107][170/200] loss=0.722387 avg=0.658380 VRAM=38.6GiB | 21.6% done | ETA(epoch): 399s + [Epoch 107][180/200] loss=0.590885 avg=0.658060 VRAM=38.5GiB | 21.6% done | ETA(epoch): 266s + [Epoch 107][190/200] loss=0.678773 avg=0.658299 VRAM=38.6GiB | 21.6% done | ETA(epoch): 133s + [Epoch 107][200/200] loss=0.678514 avg=0.659866 VRAM=38.5GiB | 21.6% done | ETA(epoch): 0s + Train loss: 0.659866 (2659.3s) ETA: 19363min + Val loss: 0.683128 [t_0.0-0.2=0.9319 t_0.2-0.4=0.7571 t_0.4-0.6=0.6364 t_0.6-0.8=0.5586 t_0.8-1.0=0.5760] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 25.3/188.5 GiB (13.4%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 108/499 (22% done) --- + [Epoch 108][10/200] loss=0.650948 avg=0.650063 VRAM=38.6GiB | 21.6% done | ETA(epoch): 2520s + [Epoch 108][20/200] loss=0.578285 avg=0.651277 VRAM=38.5GiB | 21.6% done | ETA(epoch): 2388s + [Epoch 108][30/200] loss=0.563175 avg=0.656632 VRAM=38.6GiB | 21.6% done | ETA(epoch): 2255s + [Epoch 108][40/200] loss=0.661362 avg=0.655889 VRAM=38.5GiB | 21.6% done | ETA(epoch): 2122s + [Epoch 108][50/200] loss=0.610907 avg=0.659886 VRAM=38.6GiB | 21.6% done | ETA(epoch): 1989s + [Epoch 108][60/200] loss=0.763112 avg=0.664891 VRAM=38.5GiB | 21.7% done | ETA(epoch): 1857s + [Epoch 108][70/200] loss=0.653106 avg=0.656736 VRAM=38.6GiB | 21.7% done | ETA(epoch): 1724s + [Epoch 108][80/200] loss=0.701323 avg=0.659918 VRAM=38.5GiB | 21.7% done | ETA(epoch): 1592s + [Epoch 108][90/200] loss=0.609461 avg=0.657725 VRAM=38.6GiB | 21.7% done | ETA(epoch): 1459s + [Epoch 108][100/200] loss=0.884859 avg=0.660205 VRAM=38.5GiB | 21.7% done | ETA(epoch): 1327s + [Epoch 108][110/200] loss=0.740097 avg=0.659377 VRAM=38.6GiB | 21.7% done | ETA(epoch): 1194s + [Epoch 108][120/200] loss=0.669967 avg=0.657888 VRAM=38.5GiB | 21.7% done | ETA(epoch): 1062s + [Epoch 108][130/200] loss=0.592221 avg=0.659126 VRAM=38.6GiB | 21.7% done | ETA(epoch): 929s + [Epoch 108][140/200] loss=0.576799 avg=0.659275 VRAM=38.5GiB | 21.7% done | ETA(epoch): 796s + [Epoch 108][150/200] loss=0.658834 avg=0.659838 VRAM=38.6GiB | 21.8% done | ETA(epoch): 663s + [Epoch 108][160/200] loss=0.771500 avg=0.658422 VRAM=38.5GiB | 21.8% done | ETA(epoch): 531s + [Epoch 108][170/200] loss=0.563122 avg=0.657266 VRAM=38.6GiB | 21.8% done | ETA(epoch): 398s + [Epoch 108][180/200] loss=0.598453 avg=0.659508 VRAM=38.5GiB | 21.8% done | ETA(epoch): 265s + [Epoch 108][190/200] loss=0.643082 avg=0.657579 VRAM=38.6GiB | 21.8% done | ETA(epoch): 133s + [Epoch 108][200/200] loss=0.657305 avg=0.659000 VRAM=38.5GiB | 21.8% done | ETA(epoch): 0s + Train loss: 0.659000 (2654.2s) ETA: 19310min + Val loss: 0.695176 [t_0.0-0.2=0.9278 t_0.2-0.4=0.7546 t_0.4-0.6=0.6407 t_0.6-0.8=0.5713 t_0.8-1.0=0.5664] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 25.2/188.5 GiB (13.4%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 109/499 (22% done) --- + [Epoch 109][10/200] loss=0.608634 avg=0.636310 VRAM=38.6GiB | 21.8% done | ETA(epoch): 2520s + [Epoch 109][20/200] loss=0.615553 avg=0.647627 VRAM=38.5GiB | 21.8% done | ETA(epoch): 2389s + [Epoch 109][30/200] loss=0.540015 avg=0.647160 VRAM=38.6GiB | 21.8% done | ETA(epoch): 2256s + [Epoch 109][40/200] loss=0.572413 avg=0.655840 VRAM=38.5GiB | 21.8% done | ETA(epoch): 2124s + [Epoch 109][50/200] loss=0.641630 avg=0.658830 VRAM=38.6GiB | 21.9% done | ETA(epoch): 2004s + [Epoch 109][60/200] loss=0.621669 avg=0.652733 VRAM=38.5GiB | 21.9% done | ETA(epoch): 1868s + [Epoch 109][70/200] loss=0.719348 avg=0.658157 VRAM=38.6GiB | 21.9% done | ETA(epoch): 1734s + [Epoch 109][80/200] loss=0.658216 avg=0.657831 VRAM=38.5GiB | 21.9% done | ETA(epoch): 1599s + [Epoch 109][90/200] loss=0.816657 avg=0.659304 VRAM=38.6GiB | 21.9% done | ETA(epoch): 1465s + [Epoch 109][100/200] loss=0.697318 avg=0.661262 VRAM=38.5GiB | 21.9% done | ETA(epoch): 1331s + [Epoch 109][110/200] loss=0.646642 avg=0.661587 VRAM=38.6GiB | 21.9% done | ETA(epoch): 1198s + [Epoch 109][120/200] loss=0.699354 avg=0.662906 VRAM=38.5GiB | 21.9% done | ETA(epoch): 1065s + [Epoch 109][130/200] loss=0.554832 avg=0.664157 VRAM=38.6GiB | 21.9% done | ETA(epoch): 932s + [Epoch 109][140/200] loss=0.620224 avg=0.663086 VRAM=38.5GiB | 21.9% done | ETA(epoch): 799s + [Epoch 109][150/200] loss=0.716908 avg=0.663033 VRAM=38.6GiB | 21.9% done | ETA(epoch): 666s + [Epoch 109][160/200] loss=0.655395 avg=0.663126 VRAM=38.5GiB | 22.0% done | ETA(epoch): 533s + [Epoch 109][170/200] loss=0.655491 avg=0.663276 VRAM=38.6GiB | 22.0% done | ETA(epoch): 399s + [Epoch 109][180/200] loss=0.601009 avg=0.665835 VRAM=38.5GiB | 22.0% done | ETA(epoch): 266s + [Epoch 109][190/200] loss=0.763779 avg=0.664396 VRAM=38.6GiB | 22.0% done | ETA(epoch): 133s + [Epoch 109][200/200] loss=0.581233 avg=0.662933 VRAM=38.5GiB | 22.0% done | ETA(epoch): 0s + Train loss: 0.662933 (2661.8s) ETA: 19258min + Val loss: 0.708672 [t_0.0-0.2=0.9358 t_0.2-0.4=0.7643 t_0.4-0.6=0.6604 t_0.6-0.8=0.5463 t_0.8-1.0=0.5718] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 25.2/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 110/499 (22% done) --- + [Epoch 110][10/200] loss=0.787220 avg=0.666234 VRAM=38.6GiB | 22.0% done | ETA(epoch): 2616s + [Epoch 110][20/200] loss=0.662623 avg=0.665717 VRAM=38.5GiB | 22.0% done | ETA(epoch): 2435s + [Epoch 110][30/200] loss=0.688316 avg=0.656291 VRAM=38.6GiB | 22.0% done | ETA(epoch): 2284s + [Epoch 110][40/200] loss=0.856559 avg=0.663943 VRAM=38.5GiB | 22.0% done | ETA(epoch): 2144s + [Epoch 110][50/200] loss=0.635149 avg=0.658727 VRAM=38.6GiB | 22.1% done | ETA(epoch): 2006s + [Epoch 110][60/200] loss=0.684503 avg=0.659515 VRAM=38.5GiB | 22.1% done | ETA(epoch): 1870s + [Epoch 110][70/200] loss=0.529426 avg=0.654912 VRAM=38.6GiB | 22.1% done | ETA(epoch): 1735s + [Epoch 110][80/200] loss=0.650543 avg=0.652268 VRAM=38.5GiB | 22.1% done | ETA(epoch): 1600s + [Epoch 110][90/200] loss=0.619770 avg=0.650846 VRAM=38.6GiB | 22.1% done | ETA(epoch): 1466s + [Epoch 110][100/200] loss=0.666711 avg=0.648613 VRAM=38.5GiB | 22.1% done | ETA(epoch): 1333s + [Epoch 110][110/200] loss=0.775443 avg=0.651263 VRAM=38.6GiB | 22.1% done | ETA(epoch): 1199s + [Epoch 110][120/200] loss=0.602364 avg=0.651283 VRAM=38.5GiB | 22.1% done | ETA(epoch): 1065s + [Epoch 110][130/200] loss=0.889691 avg=0.655524 VRAM=38.6GiB | 22.1% done | ETA(epoch): 932s + [Epoch 110][140/200] loss=0.599852 avg=0.656938 VRAM=38.5GiB | 22.1% done | ETA(epoch): 799s + [Epoch 110][150/200] loss=0.652312 avg=0.655879 VRAM=38.6GiB | 22.1% done | ETA(epoch): 665s + [Epoch 110][160/200] loss=0.607533 avg=0.655926 VRAM=38.5GiB | 22.2% done | ETA(epoch): 532s + [Epoch 110][170/200] loss=0.613463 avg=0.656800 VRAM=38.6GiB | 22.2% done | ETA(epoch): 399s + [Epoch 110][180/200] loss=0.617910 avg=0.655112 VRAM=38.5GiB | 22.2% done | ETA(epoch): 266s + [Epoch 110][190/200] loss=0.513618 avg=0.655767 VRAM=38.6GiB | 22.2% done | ETA(epoch): 133s + [Epoch 110][200/200] loss=0.571927 avg=0.655266 VRAM=38.5GiB | 22.2% done | ETA(epoch): 0s + Train loss: 0.655266 (2664.1s) ETA: 19205min + Val loss: 0.702331 [t_0.0-0.2=0.9440 t_0.2-0.4=0.7656 t_0.4-0.6=0.6523 t_0.6-0.8=0.5523 t_0.8-1.0=0.5798] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 25.2/188.5 GiB (13.4%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 111/499 (22% done) --- + [Epoch 111][10/200] loss=0.634404 avg=0.654568 VRAM=38.6GiB | 22.2% done | ETA(epoch): 2518s + [Epoch 111][20/200] loss=0.762843 avg=0.672321 VRAM=38.5GiB | 22.2% done | ETA(epoch): 2391s + [Epoch 111][30/200] loss=0.631136 avg=0.669958 VRAM=38.6GiB | 22.2% done | ETA(epoch): 2258s + [Epoch 111][40/200] loss=0.649046 avg=0.661905 VRAM=38.5GiB | 22.2% done | ETA(epoch): 2124s + [Epoch 111][50/200] loss=0.815178 avg=0.671411 VRAM=38.6GiB | 22.2% done | ETA(epoch): 1991s + [Epoch 111][60/200] loss=0.630680 avg=0.667912 VRAM=38.5GiB | 22.3% done | ETA(epoch): 1904s + [Epoch 111][70/200] loss=0.572712 avg=0.666029 VRAM=38.6GiB | 22.3% done | ETA(epoch): 1762s + [Epoch 111][80/200] loss=0.841651 avg=0.666254 VRAM=38.5GiB | 22.3% done | ETA(epoch): 1622s + [Epoch 111][90/200] loss=0.716641 avg=0.662742 VRAM=38.6GiB | 22.3% done | ETA(epoch): 1484s + [Epoch 111][100/200] loss=0.639397 avg=0.661711 VRAM=38.5GiB | 22.3% done | ETA(epoch): 1347s + [Epoch 111][110/200] loss=0.775537 avg=0.662566 VRAM=38.6GiB | 22.3% done | ETA(epoch): 1211s + [Epoch 111][120/200] loss=0.736856 avg=0.665300 VRAM=38.5GiB | 22.3% done | ETA(epoch): 1075s + [Epoch 111][130/200] loss=0.755003 avg=0.663487 VRAM=38.6GiB | 22.3% done | ETA(epoch): 940s + [Epoch 111][140/200] loss=0.589456 avg=0.664887 VRAM=38.5GiB | 22.3% done | ETA(epoch): 805s + [Epoch 111][150/200] loss=0.842390 avg=0.666830 VRAM=38.6GiB | 22.4% done | ETA(epoch): 670s + [Epoch 111][160/200] loss=0.654988 avg=0.666561 VRAM=38.5GiB | 22.4% done | ETA(epoch): 536s + [Epoch 111][170/200] loss=0.605184 avg=0.664644 VRAM=38.6GiB | 22.4% done | ETA(epoch): 402s + [Epoch 111][180/200] loss=0.750309 avg=0.665015 VRAM=38.5GiB | 22.4% done | ETA(epoch): 268s + [Epoch 111][190/200] loss=0.724794 avg=0.665524 VRAM=38.6GiB | 22.4% done | ETA(epoch): 134s + [Epoch 111][200/200] loss=0.677215 avg=0.665575 VRAM=38.5GiB | 22.4% done | ETA(epoch): 0s + Train loss: 0.665575 (2674.3s) ETA: 19154min + Val loss: 0.692155 [t_0.0-0.2=0.9563 t_0.2-0.4=0.7458 t_0.4-0.6=0.6384 t_0.6-0.8=0.5715 t_0.8-1.0=0.5606] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 112/499 (22% done) --- + [Epoch 112][10/200] loss=0.671251 avg=0.636152 VRAM=38.6GiB | 22.4% done | ETA(epoch): 2525s + [Epoch 112][20/200] loss=0.702002 avg=0.632082 VRAM=38.5GiB | 22.4% done | ETA(epoch): 2557s + [Epoch 112][30/200] loss=0.645076 avg=0.642307 VRAM=38.6GiB | 22.4% done | ETA(epoch): 2363s + [Epoch 112][40/200] loss=0.733319 avg=0.648654 VRAM=38.5GiB | 22.4% done | ETA(epoch): 2198s + [Epoch 112][50/200] loss=0.706733 avg=0.645213 VRAM=38.6GiB | 22.4% done | ETA(epoch): 2048s + [Epoch 112][60/200] loss=0.666716 avg=0.642815 VRAM=38.5GiB | 22.5% done | ETA(epoch): 1902s + [Epoch 112][70/200] loss=0.665860 avg=0.644211 VRAM=38.6GiB | 22.5% done | ETA(epoch): 1761s + [Epoch 112][80/200] loss=0.617417 avg=0.652791 VRAM=38.5GiB | 22.5% done | ETA(epoch): 1621s + [Epoch 112][90/200] loss=0.664849 avg=0.650165 VRAM=38.6GiB | 22.5% done | ETA(epoch): 1483s + [Epoch 112][100/200] loss=0.695459 avg=0.650339 VRAM=38.5GiB | 22.5% done | ETA(epoch): 1346s + [Epoch 112][110/200] loss=0.620358 avg=0.648204 VRAM=38.6GiB | 22.5% done | ETA(epoch): 1210s + [Epoch 112][120/200] loss=0.566582 avg=0.647963 VRAM=38.5GiB | 22.5% done | ETA(epoch): 1075s + [Epoch 112][130/200] loss=0.793886 avg=0.647296 VRAM=38.6GiB | 22.5% done | ETA(epoch): 940s + [Epoch 112][140/200] loss=0.648225 avg=0.646000 VRAM=38.5GiB | 22.5% done | ETA(epoch): 805s + [Epoch 112][150/200] loss=0.656957 avg=0.648768 VRAM=38.6GiB | 22.6% done | ETA(epoch): 670s + [Epoch 112][160/200] loss=0.651421 avg=0.649774 VRAM=38.5GiB | 22.6% done | ETA(epoch): 536s + [Epoch 112][170/200] loss=0.663657 avg=0.649953 VRAM=38.6GiB | 22.6% done | ETA(epoch): 402s + [Epoch 112][180/200] loss=0.704189 avg=0.651385 VRAM=38.5GiB | 22.6% done | ETA(epoch): 268s + [Epoch 112][190/200] loss=0.646459 avg=0.651313 VRAM=38.6GiB | 22.6% done | ETA(epoch): 134s + [Epoch 112][200/200] loss=0.604946 avg=0.651366 VRAM=38.5GiB | 22.6% done | ETA(epoch): 0s + Train loss: 0.651366 (2675.2s) ETA: 19102min + Val loss: 0.680318 [t_0.0-0.2=0.9531 t_0.2-0.4=0.7468 t_0.4-0.6=0.6404 t_0.6-0.8=0.5696 t_0.8-1.0=0.5611] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 113/499 (23% done) --- + [Epoch 113][10/200] loss=0.613979 avg=0.632577 VRAM=38.6GiB | 22.6% done | ETA(epoch): 2527s + [Epoch 113][20/200] loss=0.654749 avg=0.624811 VRAM=38.5GiB | 22.6% done | ETA(epoch): 2394s + [Epoch 113][30/200] loss=0.815754 avg=0.642370 VRAM=38.6GiB | 22.6% done | ETA(epoch): 2261s + [Epoch 113][40/200] loss=0.709734 avg=0.643896 VRAM=38.5GiB | 22.6% done | ETA(epoch): 2127s + [Epoch 113][50/200] loss=0.673551 avg=0.650052 VRAM=38.6GiB | 22.7% done | ETA(epoch): 1993s + [Epoch 113][60/200] loss=0.626115 avg=0.652639 VRAM=38.5GiB | 22.7% done | ETA(epoch): 1860s + [Epoch 113][70/200] loss=0.710344 avg=0.652422 VRAM=38.6GiB | 22.7% done | ETA(epoch): 1727s + [Epoch 113][80/200] loss=0.615196 avg=0.651177 VRAM=38.5GiB | 22.7% done | ETA(epoch): 1594s + [Epoch 113][90/200] loss=0.522565 avg=0.651257 VRAM=38.6GiB | 22.7% done | ETA(epoch): 1461s + [Epoch 113][100/200] loss=0.645697 avg=0.654394 VRAM=38.5GiB | 22.7% done | ETA(epoch): 1328s + [Epoch 113][110/200] loss=0.752748 avg=0.655698 VRAM=38.6GiB | 22.7% done | ETA(epoch): 1195s + [Epoch 113][120/200] loss=0.645200 avg=0.658033 VRAM=38.5GiB | 22.7% done | ETA(epoch): 1063s + [Epoch 113][130/200] loss=0.751770 avg=0.660386 VRAM=38.6GiB | 22.7% done | ETA(epoch): 930s + [Epoch 113][140/200] loss=0.692213 avg=0.661580 VRAM=38.5GiB | 22.7% done | ETA(epoch): 797s + [Epoch 113][150/200] loss=0.708999 avg=0.662139 VRAM=38.6GiB | 22.8% done | ETA(epoch): 664s + [Epoch 113][160/200] loss=0.703524 avg=0.662412 VRAM=38.5GiB | 22.8% done | ETA(epoch): 531s + [Epoch 113][170/200] loss=0.555630 avg=0.661998 VRAM=38.6GiB | 22.8% done | ETA(epoch): 398s + [Epoch 113][180/200] loss=0.542722 avg=0.662735 VRAM=38.5GiB | 22.8% done | ETA(epoch): 266s + [Epoch 113][190/200] loss=0.665401 avg=0.663002 VRAM=38.6GiB | 22.8% done | ETA(epoch): 133s + [Epoch 113][200/200] loss=0.579412 avg=0.664930 VRAM=38.5GiB | 22.8% done | ETA(epoch): 0s + Train loss: 0.664930 (2656.4s) ETA: 19050min + Val loss: 0.714273 [t_0.0-0.2=0.9531 t_0.2-0.4=0.7510 t_0.4-0.6=0.6310 t_0.6-0.8=0.5781 t_0.8-1.0=0.5790] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 114/499 (23% done) --- + [Epoch 114][10/200] loss=0.681370 avg=0.658345 VRAM=38.6GiB | 22.8% done | ETA(epoch): 2526s + [Epoch 114][20/200] loss=0.572041 avg=0.673343 VRAM=38.5GiB | 22.8% done | ETA(epoch): 2391s + [Epoch 114][30/200] loss=0.645197 avg=0.674150 VRAM=38.6GiB | 22.8% done | ETA(epoch): 2258s + [Epoch 114][40/200] loss=0.709979 avg=0.671020 VRAM=38.5GiB | 22.8% done | ETA(epoch): 2124s + [Epoch 114][50/200] loss=0.693419 avg=0.664888 VRAM=38.6GiB | 22.9% done | ETA(epoch): 1991s + [Epoch 114][60/200] loss=0.669481 avg=0.665053 VRAM=38.5GiB | 22.9% done | ETA(epoch): 1859s + [Epoch 114][70/200] loss=0.687901 avg=0.666080 VRAM=38.6GiB | 22.9% done | ETA(epoch): 1726s + [Epoch 114][80/200] loss=0.669520 avg=0.666958 VRAM=38.5GiB | 22.9% done | ETA(epoch): 1593s + [Epoch 114][90/200] loss=0.750839 avg=0.670756 VRAM=38.6GiB | 22.9% done | ETA(epoch): 1461s + [Epoch 114][100/200] loss=0.621650 avg=0.667974 VRAM=38.5GiB | 22.9% done | ETA(epoch): 1328s + [Epoch 114][110/200] loss=0.540828 avg=0.668243 VRAM=38.6GiB | 22.9% done | ETA(epoch): 1195s + [Epoch 114][120/200] loss=0.877669 avg=0.671770 VRAM=38.5GiB | 22.9% done | ETA(epoch): 1062s + [Epoch 114][130/200] loss=0.677483 avg=0.669109 VRAM=38.6GiB | 22.9% done | ETA(epoch): 929s + [Epoch 114][140/200] loss=0.631878 avg=0.669251 VRAM=38.5GiB | 22.9% done | ETA(epoch): 797s + [Epoch 114][150/200] loss=0.657875 avg=0.668904 VRAM=38.6GiB | 22.9% done | ETA(epoch): 664s + [Epoch 114][160/200] loss=0.634455 avg=0.668275 VRAM=38.5GiB | 23.0% done | ETA(epoch): 531s + [Epoch 114][170/200] loss=0.562835 avg=0.666796 VRAM=38.6GiB | 23.0% done | ETA(epoch): 398s + [Epoch 114][180/200] loss=0.615915 avg=0.666587 VRAM=38.5GiB | 23.0% done | ETA(epoch): 266s + [Epoch 114][190/200] loss=0.764032 avg=0.667217 VRAM=38.6GiB | 23.0% done | ETA(epoch): 133s + [Epoch 114][200/200] loss=0.635912 avg=0.666014 VRAM=38.5GiB | 23.0% done | ETA(epoch): 0s + Train loss: 0.666014 (2656.3s) ETA: 18997min + Val loss: 0.685882 [t_0.0-0.2=0.9303 t_0.2-0.4=0.7751 t_0.4-0.6=0.6340 t_0.6-0.8=0.5763 t_0.8-1.0=0.5689] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 115/499 (23% done) --- + [Epoch 115][10/200] loss=0.835154 avg=0.649074 VRAM=38.6GiB | 23.0% done | ETA(epoch): 2527s + [Epoch 115][20/200] loss=0.599013 avg=0.649978 VRAM=38.5GiB | 23.0% done | ETA(epoch): 2391s + [Epoch 115][30/200] loss=0.702359 avg=0.650545 VRAM=38.6GiB | 23.0% done | ETA(epoch): 2260s + [Epoch 115][40/200] loss=0.717058 avg=0.656539 VRAM=38.5GiB | 23.0% done | ETA(epoch): 2126s + [Epoch 115][50/200] loss=0.586789 avg=0.660600 VRAM=38.6GiB | 23.1% done | ETA(epoch): 1994s + [Epoch 115][60/200] loss=0.636889 avg=0.661726 VRAM=38.5GiB | 23.1% done | ETA(epoch): 1860s + [Epoch 115][70/200] loss=0.687740 avg=0.665594 VRAM=38.6GiB | 23.1% done | ETA(epoch): 1728s + [Epoch 115][80/200] loss=0.821134 avg=0.672580 VRAM=38.5GiB | 23.1% done | ETA(epoch): 1594s + [Epoch 115][90/200] loss=0.620986 avg=0.671035 VRAM=38.6GiB | 23.1% done | ETA(epoch): 1465s + [Epoch 115][100/200] loss=0.638853 avg=0.670630 VRAM=38.5GiB | 23.1% done | ETA(epoch): 1331s + [Epoch 115][110/200] loss=0.732224 avg=0.674074 VRAM=38.6GiB | 23.1% done | ETA(epoch): 1198s + [Epoch 115][120/200] loss=0.688646 avg=0.675261 VRAM=38.5GiB | 23.1% done | ETA(epoch): 1065s + [Epoch 115][130/200] loss=0.681381 avg=0.672716 VRAM=38.6GiB | 23.1% done | ETA(epoch): 931s + [Epoch 115][140/200] loss=0.589178 avg=0.672485 VRAM=38.5GiB | 23.1% done | ETA(epoch): 798s + [Epoch 115][150/200] loss=0.696019 avg=0.674233 VRAM=38.6GiB | 23.2% done | ETA(epoch): 665s + [Epoch 115][160/200] loss=0.839124 avg=0.673152 VRAM=38.5GiB | 23.2% done | ETA(epoch): 532s + [Epoch 115][170/200] loss=0.698607 avg=0.673185 VRAM=38.6GiB | 23.2% done | ETA(epoch): 399s + [Epoch 115][180/200] loss=0.590699 avg=0.670820 VRAM=38.5GiB | 23.2% done | ETA(epoch): 266s + [Epoch 115][190/200] loss=0.590755 avg=0.668142 VRAM=38.6GiB | 23.2% done | ETA(epoch): 133s + [Epoch 115][200/200] loss=0.792170 avg=0.668075 VRAM=38.5GiB | 23.2% done | ETA(epoch): 0s + Train loss: 0.668075 (2660.0s) ETA: 18945min + Val loss: 0.685575 [t_0.0-0.2=0.9319 t_0.2-0.4=0.7595 t_0.4-0.6=0.6524 t_0.6-0.8=0.5574 t_0.8-1.0=0.5636] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 23.7/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 116/499 (23% done) --- + [Epoch 116][10/200] loss=0.586439 avg=0.652181 VRAM=38.6GiB | 23.2% done | ETA(epoch): 2519s + [Epoch 116][20/200] loss=0.662471 avg=0.668123 VRAM=38.5GiB | 23.2% done | ETA(epoch): 2391s + [Epoch 116][30/200] loss=0.673125 avg=0.668432 VRAM=38.6GiB | 23.2% done | ETA(epoch): 2257s + [Epoch 116][40/200] loss=0.649639 avg=0.666712 VRAM=38.5GiB | 23.2% done | ETA(epoch): 2124s + [Epoch 116][50/200] loss=0.573504 avg=0.666421 VRAM=38.6GiB | 23.2% done | ETA(epoch): 1991s + [Epoch 116][60/200] loss=0.592860 avg=0.667882 VRAM=38.5GiB | 23.3% done | ETA(epoch): 1868s + [Epoch 116][70/200] loss=0.607038 avg=0.665589 VRAM=38.6GiB | 23.3% done | ETA(epoch): 1734s + [Epoch 116][80/200] loss=0.660143 avg=0.665207 VRAM=38.5GiB | 23.3% done | ETA(epoch): 1599s + [Epoch 116][90/200] loss=0.578555 avg=0.663262 VRAM=38.6GiB | 23.3% done | ETA(epoch): 1465s + [Epoch 116][100/200] loss=0.537349 avg=0.661231 VRAM=38.5GiB | 23.3% done | ETA(epoch): 1332s + [Epoch 116][110/200] loss=0.612441 avg=0.659968 VRAM=38.6GiB | 23.3% done | ETA(epoch): 1198s + [Epoch 116][120/200] loss=0.606244 avg=0.657092 VRAM=38.5GiB | 23.3% done | ETA(epoch): 1065s + [Epoch 116][130/200] loss=0.624104 avg=0.654661 VRAM=38.6GiB | 23.3% done | ETA(epoch): 932s + [Epoch 116][140/200] loss=0.648995 avg=0.655522 VRAM=38.5GiB | 23.3% done | ETA(epoch): 799s + [Epoch 116][150/200] loss=0.680061 avg=0.654461 VRAM=38.6GiB | 23.4% done | ETA(epoch): 667s + [Epoch 116][160/200] loss=0.780030 avg=0.655069 VRAM=38.5GiB | 23.4% done | ETA(epoch): 534s + [Epoch 116][170/200] loss=0.617015 avg=0.654450 VRAM=38.6GiB | 23.4% done | ETA(epoch): 400s + [Epoch 116][180/200] loss=0.598901 avg=0.655298 VRAM=38.5GiB | 23.4% done | ETA(epoch): 267s + [Epoch 116][190/200] loss=0.685571 avg=0.657720 VRAM=38.6GiB | 23.4% done | ETA(epoch): 133s + [Epoch 116][200/200] loss=0.649416 avg=0.658188 VRAM=38.5GiB | 23.4% done | ETA(epoch): 0s + Train loss: 0.658188 (2665.6s) ETA: 18893min + Val loss: 0.693708 [t_0.0-0.2=0.9408 t_0.2-0.4=0.7471 t_0.4-0.6=0.6475 t_0.6-0.8=0.5758 t_0.8-1.0=0.5580] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 117/499 (23% done) --- + [Epoch 117][10/200] loss=0.750907 avg=0.696962 VRAM=38.6GiB | 23.4% done | ETA(epoch): 2521s + [Epoch 117][20/200] loss=0.650271 avg=0.685224 VRAM=38.5GiB | 23.4% done | ETA(epoch): 2390s + [Epoch 117][30/200] loss=0.724061 avg=0.687762 VRAM=38.6GiB | 23.4% done | ETA(epoch): 2258s + [Epoch 117][40/200] loss=0.671903 avg=0.692659 VRAM=38.5GiB | 23.4% done | ETA(epoch): 2125s + [Epoch 117][50/200] loss=0.652042 avg=0.680692 VRAM=38.6GiB | 23.4% done | ETA(epoch): 1992s + [Epoch 117][60/200] loss=0.587805 avg=0.679749 VRAM=38.5GiB | 23.5% done | ETA(epoch): 1859s + [Epoch 117][70/200] loss=0.763406 avg=0.673559 VRAM=38.6GiB | 23.5% done | ETA(epoch): 1727s + [Epoch 117][80/200] loss=0.724187 avg=0.673454 VRAM=38.5GiB | 23.5% done | ETA(epoch): 1593s + [Epoch 117][90/200] loss=0.896728 avg=0.676406 VRAM=38.6GiB | 23.5% done | ETA(epoch): 1461s + [Epoch 117][100/200] loss=0.685040 avg=0.672211 VRAM=38.5GiB | 23.5% done | ETA(epoch): 1328s + [Epoch 117][110/200] loss=0.555952 avg=0.670079 VRAM=38.6GiB | 23.5% done | ETA(epoch): 1195s + [Epoch 117][120/200] loss=0.623774 avg=0.666457 VRAM=38.5GiB | 23.5% done | ETA(epoch): 1063s + [Epoch 117][130/200] loss=0.554638 avg=0.664372 VRAM=38.6GiB | 23.5% done | ETA(epoch): 930s + [Epoch 117][140/200] loss=0.620210 avg=0.664739 VRAM=38.5GiB | 23.5% done | ETA(epoch): 797s + [Epoch 117][150/200] loss=0.617443 avg=0.661585 VRAM=38.6GiB | 23.5% done | ETA(epoch): 664s + [Epoch 117][160/200] loss=0.579188 avg=0.659525 VRAM=38.5GiB | 23.6% done | ETA(epoch): 531s + [Epoch 117][170/200] loss=0.734826 avg=0.659377 VRAM=38.6GiB | 23.6% done | ETA(epoch): 398s + [Epoch 117][180/200] loss=0.742492 avg=0.661419 VRAM=38.5GiB | 23.6% done | ETA(epoch): 266s + [Epoch 117][190/200] loss=0.610515 avg=0.660954 VRAM=38.6GiB | 23.6% done | ETA(epoch): 133s + [Epoch 117][200/200] loss=0.675011 avg=0.661777 VRAM=38.5GiB | 23.6% done | ETA(epoch): 0s + Train loss: 0.661777 (2660.0s) ETA: 18841min + Val loss: 0.695450 [t_0.0-0.2=0.9576 t_0.2-0.4=0.7708 t_0.4-0.6=0.6420 t_0.6-0.8=0.5510 t_0.8-1.0=0.5688] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 118/499 (24% done) --- + [Epoch 118][10/200] loss=0.558460 avg=0.698658 VRAM=38.6GiB | 23.6% done | ETA(epoch): 2525s + [Epoch 118][20/200] loss=0.669827 avg=0.680143 VRAM=38.5GiB | 23.6% done | ETA(epoch): 2390s + [Epoch 118][30/200] loss=0.634319 avg=0.683006 VRAM=38.6GiB | 23.6% done | ETA(epoch): 2259s + [Epoch 118][40/200] loss=0.624764 avg=0.674004 VRAM=38.5GiB | 23.6% done | ETA(epoch): 2126s + [Epoch 118][50/200] loss=0.718044 avg=0.671183 VRAM=38.6GiB | 23.6% done | ETA(epoch): 1992s + [Epoch 118][60/200] loss=0.669933 avg=0.672034 VRAM=38.5GiB | 23.7% done | ETA(epoch): 1859s + [Epoch 118][70/200] loss=0.631594 avg=0.667809 VRAM=38.6GiB | 23.7% done | ETA(epoch): 1727s + [Epoch 118][80/200] loss=0.663045 avg=0.661007 VRAM=38.5GiB | 23.7% done | ETA(epoch): 1594s + [Epoch 118][90/200] loss=0.679559 avg=0.661244 VRAM=38.6GiB | 23.7% done | ETA(epoch): 1461s + [Epoch 118][100/200] loss=0.567593 avg=0.661047 VRAM=38.5GiB | 23.7% done | ETA(epoch): 1328s + [Epoch 118][110/200] loss=0.668038 avg=0.660533 VRAM=38.6GiB | 23.7% done | ETA(epoch): 1196s + [Epoch 118][120/200] loss=0.612102 avg=0.662821 VRAM=38.5GiB | 23.7% done | ETA(epoch): 1063s + [Epoch 118][130/200] loss=0.844412 avg=0.665890 VRAM=38.6GiB | 23.7% done | ETA(epoch): 930s + [Epoch 118][140/200] loss=0.818356 avg=0.665974 VRAM=38.5GiB | 23.7% done | ETA(epoch): 797s + [Epoch 118][150/200] loss=0.666103 avg=0.666524 VRAM=38.6GiB | 23.8% done | ETA(epoch): 664s + [Epoch 118][160/200] loss=0.607230 avg=0.664854 VRAM=38.5GiB | 23.8% done | ETA(epoch): 531s + [Epoch 118][170/200] loss=0.632001 avg=0.665864 VRAM=38.6GiB | 23.8% done | ETA(epoch): 398s + [Epoch 118][180/200] loss=0.730394 avg=0.668230 VRAM=38.5GiB | 23.8% done | ETA(epoch): 266s + [Epoch 118][190/200] loss=0.744779 avg=0.667124 VRAM=38.6GiB | 23.8% done | ETA(epoch): 133s + [Epoch 118][200/200] loss=0.779281 avg=0.666295 VRAM=38.5GiB | 23.8% done | ETA(epoch): 0s + Train loss: 0.666295 (2655.9s) ETA: 18789min + Val loss: 0.685072 [t_0.0-0.2=0.9344 t_0.2-0.4=0.7525 t_0.4-0.6=0.6518 t_0.6-0.8=0.5543 t_0.8-1.0=0.5748] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 119/499 (24% done) --- + [Epoch 119][10/200] loss=0.598831 avg=0.662621 VRAM=38.6GiB | 23.8% done | ETA(epoch): 2521s + [Epoch 119][20/200] loss=0.627509 avg=0.681745 VRAM=38.5GiB | 23.8% done | ETA(epoch): 2387s + [Epoch 119][30/200] loss=0.583426 avg=0.659864 VRAM=38.6GiB | 23.8% done | ETA(epoch): 2256s + [Epoch 119][40/200] loss=0.675301 avg=0.660825 VRAM=38.5GiB | 23.8% done | ETA(epoch): 2123s + [Epoch 119][50/200] loss=0.658539 avg=0.649088 VRAM=38.6GiB | 23.8% done | ETA(epoch): 1990s + [Epoch 119][60/200] loss=0.683054 avg=0.651275 VRAM=38.5GiB | 23.9% done | ETA(epoch): 1858s + [Epoch 119][70/200] loss=0.824250 avg=0.653071 VRAM=38.6GiB | 23.9% done | ETA(epoch): 1725s + [Epoch 119][80/200] loss=0.812238 avg=0.657474 VRAM=38.5GiB | 23.9% done | ETA(epoch): 1592s + [Epoch 119][90/200] loss=0.683194 avg=0.660610 VRAM=38.6GiB | 23.9% done | ETA(epoch): 1460s + [Epoch 119][100/200] loss=0.900813 avg=0.660561 VRAM=38.5GiB | 23.9% done | ETA(epoch): 1327s + [Epoch 119][110/200] loss=0.609662 avg=0.660925 VRAM=38.6GiB | 23.9% done | ETA(epoch): 1194s + [Epoch 119][120/200] loss=0.644235 avg=0.659888 VRAM=38.5GiB | 23.9% done | ETA(epoch): 1064s + [Epoch 119][130/200] loss=0.740148 avg=0.659169 VRAM=38.6GiB | 23.9% done | ETA(epoch): 931s + [Epoch 119][140/200] loss=0.517749 avg=0.658288 VRAM=38.5GiB | 23.9% done | ETA(epoch): 798s + [Epoch 119][150/200] loss=0.616994 avg=0.659742 VRAM=38.6GiB | 23.9% done | ETA(epoch): 665s + [Epoch 119][160/200] loss=0.616015 avg=0.659027 VRAM=38.5GiB | 24.0% done | ETA(epoch): 532s + [Epoch 119][170/200] loss=0.602723 avg=0.659298 VRAM=38.6GiB | 24.0% done | ETA(epoch): 399s + [Epoch 119][180/200] loss=0.685403 avg=0.659070 VRAM=38.5GiB | 24.0% done | ETA(epoch): 266s + [Epoch 119][190/200] loss=0.723693 avg=0.660929 VRAM=38.6GiB | 24.0% done | ETA(epoch): 133s + [Epoch 119][200/200] loss=0.753271 avg=0.663507 VRAM=38.5GiB | 24.0% done | ETA(epoch): 0s + Train loss: 0.663507 (2658.3s) ETA: 18737min + Val loss: 0.722493 [t_0.0-0.2=0.9508 t_0.2-0.4=0.7784 t_0.4-0.6=0.6341 t_0.6-0.8=0.5670 t_0.8-1.0=0.5648] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 120/499 (24% done) --- + [Epoch 120][10/200] loss=0.553191 avg=0.649357 VRAM=38.6GiB | 24.0% done | ETA(epoch): 2516s + [Epoch 120][20/200] loss=0.559327 avg=0.652324 VRAM=38.5GiB | 24.0% done | ETA(epoch): 2387s + [Epoch 120][30/200] loss=0.599713 avg=0.658510 VRAM=38.6GiB | 24.0% done | ETA(epoch): 2256s + [Epoch 120][40/200] loss=0.605937 avg=0.655550 VRAM=38.5GiB | 24.0% done | ETA(epoch): 2124s + [Epoch 120][50/200] loss=0.671316 avg=0.656650 VRAM=38.6GiB | 24.1% done | ETA(epoch): 1991s + [Epoch 120][60/200] loss=0.640968 avg=0.659907 VRAM=38.5GiB | 24.1% done | ETA(epoch): 1859s + [Epoch 120][70/200] loss=0.649025 avg=0.663529 VRAM=38.6GiB | 24.1% done | ETA(epoch): 1727s + [Epoch 120][80/200] loss=0.722082 avg=0.664159 VRAM=38.5GiB | 24.1% done | ETA(epoch): 1594s + [Epoch 120][90/200] loss=0.556931 avg=0.666354 VRAM=38.6GiB | 24.1% done | ETA(epoch): 1461s + [Epoch 120][100/200] loss=0.652334 avg=0.669775 VRAM=38.5GiB | 24.1% done | ETA(epoch): 1328s + [Epoch 120][110/200] loss=0.648937 avg=0.668075 VRAM=38.6GiB | 24.1% done | ETA(epoch): 1195s + [Epoch 120][120/200] loss=0.674202 avg=0.666645 VRAM=38.5GiB | 24.1% done | ETA(epoch): 1062s + [Epoch 120][130/200] loss=0.581549 avg=0.666933 VRAM=38.6GiB | 24.1% done | ETA(epoch): 930s + [Epoch 120][140/200] loss=0.546532 avg=0.664591 VRAM=38.5GiB | 24.1% done | ETA(epoch): 797s + [Epoch 120][150/200] loss=0.613052 avg=0.665129 VRAM=38.6GiB | 24.1% done | ETA(epoch): 664s + [Epoch 120][160/200] loss=0.908191 avg=0.664644 VRAM=38.5GiB | 24.2% done | ETA(epoch): 531s + [Epoch 120][170/200] loss=0.564692 avg=0.663618 VRAM=38.6GiB | 24.2% done | ETA(epoch): 398s + [Epoch 120][180/200] loss=0.580166 avg=0.663404 VRAM=38.5GiB | 24.2% done | ETA(epoch): 266s + [Epoch 120][190/200] loss=0.695902 avg=0.664435 VRAM=38.6GiB | 24.2% done | ETA(epoch): 133s + [Epoch 120][200/200] loss=0.624706 avg=0.664670 VRAM=38.5GiB | 24.2% done | ETA(epoch): 0s + Train loss: 0.664670 (2659.4s) ETA: 18686min + Val loss: 0.693594 [t_0.0-0.2=0.9169 t_0.2-0.4=0.7371 t_0.4-0.6=0.6564 t_0.6-0.8=0.5625 t_0.8-1.0=0.5578] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 121/499 (24% done) --- + [Epoch 121][10/200] loss=0.635222 avg=0.659949 VRAM=38.6GiB | 24.2% done | ETA(epoch): 2519s + [Epoch 121][20/200] loss=0.758275 avg=0.647042 VRAM=38.5GiB | 24.2% done | ETA(epoch): 2389s + [Epoch 121][30/200] loss=0.560339 avg=0.664558 VRAM=38.6GiB | 24.2% done | ETA(epoch): 2256s + [Epoch 121][40/200] loss=0.687860 avg=0.658704 VRAM=38.5GiB | 24.2% done | ETA(epoch): 2124s + [Epoch 121][50/200] loss=0.613883 avg=0.654837 VRAM=38.6GiB | 24.2% done | ETA(epoch): 1991s + [Epoch 121][60/200] loss=0.611764 avg=0.654353 VRAM=38.5GiB | 24.3% done | ETA(epoch): 1858s + [Epoch 121][70/200] loss=0.679653 avg=0.659362 VRAM=38.6GiB | 24.3% done | ETA(epoch): 1726s + [Epoch 121][80/200] loss=0.636181 avg=0.656624 VRAM=38.5GiB | 24.3% done | ETA(epoch): 1593s + [Epoch 121][90/200] loss=0.686872 avg=0.658030 VRAM=38.6GiB | 24.3% done | ETA(epoch): 1460s + [Epoch 121][100/200] loss=0.698424 avg=0.656843 VRAM=38.5GiB | 24.3% done | ETA(epoch): 1327s + [Epoch 121][110/200] loss=0.602687 avg=0.658202 VRAM=38.6GiB | 24.3% done | ETA(epoch): 1195s + [Epoch 121][120/200] loss=0.769527 avg=0.662835 VRAM=38.5GiB | 24.3% done | ETA(epoch): 1062s + [Epoch 121][130/200] loss=0.630174 avg=0.663992 VRAM=38.6GiB | 24.3% done | ETA(epoch): 929s + [Epoch 121][140/200] loss=0.572989 avg=0.662937 VRAM=38.5GiB | 24.3% done | ETA(epoch): 796s + [Epoch 121][150/200] loss=0.648585 avg=0.661602 VRAM=38.6GiB | 24.3% done | ETA(epoch): 664s + [Epoch 121][160/200] loss=0.603807 avg=0.659758 VRAM=38.5GiB | 24.4% done | ETA(epoch): 531s + [Epoch 121][170/200] loss=0.677166 avg=0.656978 VRAM=38.6GiB | 24.4% done | ETA(epoch): 398s + [Epoch 121][180/200] loss=0.725455 avg=0.659080 VRAM=38.5GiB | 24.4% done | ETA(epoch): 266s + [Epoch 121][190/200] loss=0.616881 avg=0.658424 VRAM=38.6GiB | 24.4% done | ETA(epoch): 133s + [Epoch 121][200/200] loss=0.600959 avg=0.659747 VRAM=38.5GiB | 24.4% done | ETA(epoch): 0s + Train loss: 0.659747 (2655.1s) ETA: 18633min + Val loss: 0.682795 [t_0.0-0.2=0.9052 t_0.2-0.4=0.7502 t_0.4-0.6=0.6432 t_0.6-0.8=0.5556 t_0.8-1.0=0.5798] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 122/499 (24% done) --- + [Epoch 122][10/200] loss=0.577720 avg=0.624139 VRAM=38.6GiB | 24.4% done | ETA(epoch): 2573s + [Epoch 122][20/200] loss=0.746463 avg=0.642431 VRAM=38.5GiB | 24.4% done | ETA(epoch): 2413s + [Epoch 122][30/200] loss=0.586895 avg=0.644875 VRAM=38.6GiB | 24.4% done | ETA(epoch): 2271s + [Epoch 122][40/200] loss=0.530792 avg=0.645449 VRAM=38.5GiB | 24.4% done | ETA(epoch): 2133s + [Epoch 122][50/200] loss=0.631990 avg=0.652556 VRAM=38.6GiB | 24.4% done | ETA(epoch): 1999s + [Epoch 122][60/200] loss=0.759428 avg=0.652361 VRAM=38.5GiB | 24.5% done | ETA(epoch): 1864s + [Epoch 122][70/200] loss=0.790127 avg=0.658221 VRAM=38.6GiB | 24.5% done | ETA(epoch): 1730s + [Epoch 122][80/200] loss=0.654153 avg=0.653852 VRAM=38.5GiB | 24.5% done | ETA(epoch): 1596s + [Epoch 122][90/200] loss=0.633867 avg=0.654939 VRAM=38.6GiB | 24.5% done | ETA(epoch): 1463s + [Epoch 122][100/200] loss=0.716402 avg=0.654901 VRAM=38.5GiB | 24.5% done | ETA(epoch): 1334s + [Epoch 122][110/200] loss=0.625663 avg=0.657093 VRAM=38.6GiB | 24.5% done | ETA(epoch): 1200s + [Epoch 122][120/200] loss=0.663308 avg=0.656238 VRAM=38.5GiB | 24.5% done | ETA(epoch): 1066s + [Epoch 122][130/200] loss=0.765144 avg=0.658638 VRAM=38.6GiB | 24.5% done | ETA(epoch): 933s + [Epoch 122][140/200] loss=0.593100 avg=0.662539 VRAM=38.5GiB | 24.5% done | ETA(epoch): 799s + [Epoch 122][150/200] loss=0.521495 avg=0.660280 VRAM=38.6GiB | 24.6% done | ETA(epoch): 666s + [Epoch 122][160/200] loss=0.646295 avg=0.661247 VRAM=38.5GiB | 24.6% done | ETA(epoch): 533s + [Epoch 122][170/200] loss=0.758004 avg=0.661669 VRAM=38.6GiB | 24.6% done | ETA(epoch): 399s + [Epoch 122][180/200] loss=0.568955 avg=0.660378 VRAM=38.5GiB | 24.6% done | ETA(epoch): 266s + [Epoch 122][190/200] loss=0.704924 avg=0.661021 VRAM=38.6GiB | 24.6% done | ETA(epoch): 133s + [Epoch 122][200/200] loss=0.614942 avg=0.660337 VRAM=38.5GiB | 24.6% done | ETA(epoch): 0s + Train loss: 0.660337 (2664.3s) ETA: 18582min + Val loss: 0.707903 [t_0.0-0.2=0.9500 t_0.2-0.4=0.7414 t_0.4-0.6=0.6494 t_0.6-0.8=0.5746 t_0.8-1.0=0.5574] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 123/499 (25% done) --- + [Epoch 123][10/200] loss=0.606156 avg=0.657640 VRAM=38.6GiB | 24.6% done | ETA(epoch): 2524s + [Epoch 123][20/200] loss=0.628141 avg=0.664568 VRAM=38.5GiB | 24.6% done | ETA(epoch): 2389s + [Epoch 123][30/200] loss=0.631487 avg=0.654121 VRAM=38.6GiB | 24.6% done | ETA(epoch): 2256s + [Epoch 123][40/200] loss=0.538123 avg=0.648448 VRAM=38.5GiB | 24.6% done | ETA(epoch): 2123s + [Epoch 123][50/200] loss=0.799724 avg=0.649724 VRAM=38.6GiB | 24.6% done | ETA(epoch): 1990s + [Epoch 123][60/200] loss=0.562072 avg=0.651797 VRAM=38.5GiB | 24.7% done | ETA(epoch): 1864s + [Epoch 123][70/200] loss=0.564501 avg=0.652170 VRAM=38.6GiB | 24.7% done | ETA(epoch): 1730s + [Epoch 123][80/200] loss=0.654377 avg=0.650888 VRAM=38.5GiB | 24.7% done | ETA(epoch): 1596s + [Epoch 123][90/200] loss=0.622911 avg=0.651182 VRAM=38.6GiB | 24.7% done | ETA(epoch): 1463s + [Epoch 123][100/200] loss=0.540396 avg=0.651221 VRAM=38.5GiB | 24.7% done | ETA(epoch): 1330s + [Epoch 123][110/200] loss=0.691073 avg=0.651598 VRAM=38.6GiB | 24.7% done | ETA(epoch): 1197s + [Epoch 123][120/200] loss=0.775965 avg=0.655986 VRAM=38.5GiB | 24.7% done | ETA(epoch): 1064s + [Epoch 123][130/200] loss=0.786738 avg=0.654427 VRAM=38.6GiB | 24.7% done | ETA(epoch): 931s + [Epoch 123][140/200] loss=0.649781 avg=0.655578 VRAM=38.5GiB | 24.7% done | ETA(epoch): 798s + [Epoch 123][150/200] loss=0.645192 avg=0.654370 VRAM=38.6GiB | 24.8% done | ETA(epoch): 664s + [Epoch 123][160/200] loss=0.710896 avg=0.655790 VRAM=38.5GiB | 24.8% done | ETA(epoch): 532s + [Epoch 123][170/200] loss=0.715129 avg=0.655362 VRAM=38.6GiB | 24.8% done | ETA(epoch): 399s + [Epoch 123][180/200] loss=0.710218 avg=0.653597 VRAM=38.5GiB | 24.8% done | ETA(epoch): 266s + [Epoch 123][190/200] loss=0.709182 avg=0.654275 VRAM=38.6GiB | 24.8% done | ETA(epoch): 133s + [Epoch 123][200/200] loss=0.532050 avg=0.653656 VRAM=38.5GiB | 24.8% done | ETA(epoch): 0s + Train loss: 0.653656 (2660.2s) ETA: 18530min + Val loss: 0.702040 [t_0.0-0.2=0.9409 t_0.2-0.4=0.7586 t_0.4-0.6=0.6280 t_0.6-0.8=0.5663 t_0.8-1.0=0.5643] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 124/499 (25% done) --- + [Epoch 124][10/200] loss=0.579991 avg=0.669533 VRAM=38.6GiB | 24.8% done | ETA(epoch): 2520s + [Epoch 124][20/200] loss=0.568649 avg=0.663608 VRAM=38.5GiB | 24.8% done | ETA(epoch): 2389s + [Epoch 124][30/200] loss=0.525657 avg=0.666493 VRAM=38.6GiB | 24.8% done | ETA(epoch): 2258s + [Epoch 124][40/200] loss=0.687971 avg=0.670480 VRAM=38.5GiB | 24.8% done | ETA(epoch): 2125s + [Epoch 124][50/200] loss=0.610892 avg=0.662831 VRAM=38.6GiB | 24.9% done | ETA(epoch): 1992s + [Epoch 124][60/200] loss=0.684773 avg=0.668028 VRAM=38.5GiB | 24.9% done | ETA(epoch): 1859s + [Epoch 124][70/200] loss=0.767439 avg=0.667727 VRAM=38.6GiB | 24.9% done | ETA(epoch): 1727s + [Epoch 124][80/200] loss=0.668722 avg=0.674770 VRAM=38.5GiB | 24.9% done | ETA(epoch): 1593s + [Epoch 124][90/200] loss=0.568234 avg=0.669573 VRAM=38.6GiB | 24.9% done | ETA(epoch): 1460s + [Epoch 124][100/200] loss=0.630316 avg=0.666561 VRAM=38.5GiB | 24.9% done | ETA(epoch): 1328s + [Epoch 124][110/200] loss=0.669891 avg=0.665641 VRAM=38.6GiB | 24.9% done | ETA(epoch): 1195s + [Epoch 124][120/200] loss=0.619195 avg=0.665419 VRAM=38.5GiB | 24.9% done | ETA(epoch): 1062s + [Epoch 124][130/200] loss=0.738713 avg=0.666309 VRAM=38.6GiB | 24.9% done | ETA(epoch): 930s + [Epoch 124][140/200] loss=0.571169 avg=0.662899 VRAM=38.5GiB | 24.9% done | ETA(epoch): 797s + [Epoch 124][150/200] loss=0.670481 avg=0.665092 VRAM=38.6GiB | 24.9% done | ETA(epoch): 664s + [Epoch 124][160/200] loss=0.796055 avg=0.663536 VRAM=38.5GiB | 25.0% done | ETA(epoch): 531s + [Epoch 124][170/200] loss=0.764691 avg=0.665095 VRAM=38.6GiB | 25.0% done | ETA(epoch): 398s + [Epoch 124][180/200] loss=0.543379 avg=0.665397 VRAM=38.5GiB | 25.0% done | ETA(epoch): 266s + [Epoch 124][190/200] loss=0.681437 avg=0.662845 VRAM=38.6GiB | 25.0% done | ETA(epoch): 133s + [Epoch 124][200/200] loss=0.727580 avg=0.663047 VRAM=38.5GiB | 25.0% done | ETA(epoch): 0s + Train loss: 0.663047 (2655.6s) ETA: 18478min + Val loss: 0.686231 [t_0.0-0.2=0.9186 t_0.2-0.4=0.7376 t_0.4-0.6=0.6393 t_0.6-0.8=0.5731 t_0.8-1.0=0.5646] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 125/499 (25% done) --- + [Epoch 125][10/200] loss=0.633522 avg=0.665291 VRAM=38.6GiB | 25.0% done | ETA(epoch): 2520s + [Epoch 125][20/200] loss=0.735768 avg=0.663651 VRAM=38.5GiB | 25.0% done | ETA(epoch): 2387s + [Epoch 125][30/200] loss=0.676535 avg=0.654039 VRAM=38.6GiB | 25.0% done | ETA(epoch): 2256s + [Epoch 125][40/200] loss=0.751846 avg=0.664876 VRAM=38.5GiB | 25.0% done | ETA(epoch): 2124s + [Epoch 125][50/200] loss=0.571522 avg=0.657560 VRAM=38.6GiB | 25.1% done | ETA(epoch): 1992s + [Epoch 125][60/200] loss=0.610503 avg=0.656692 VRAM=38.5GiB | 25.1% done | ETA(epoch): 1859s + [Epoch 125][70/200] loss=0.653333 avg=0.656858 VRAM=38.6GiB | 25.1% done | ETA(epoch): 1727s + [Epoch 125][80/200] loss=0.593800 avg=0.656525 VRAM=38.5GiB | 25.1% done | ETA(epoch): 1602s + [Epoch 125][90/200] loss=0.635443 avg=0.658532 VRAM=38.6GiB | 25.1% done | ETA(epoch): 1467s + [Epoch 125][100/200] loss=0.604367 avg=0.654690 VRAM=38.5GiB | 25.1% done | ETA(epoch): 1333s + [Epoch 125][110/200] loss=0.715182 avg=0.655871 VRAM=38.6GiB | 25.1% done | ETA(epoch): 1200s + [Epoch 125][120/200] loss=0.679365 avg=0.656974 VRAM=38.5GiB | 25.1% done | ETA(epoch): 1066s + [Epoch 125][130/200] loss=0.623766 avg=0.656151 VRAM=38.6GiB | 25.1% done | ETA(epoch): 933s + [Epoch 125][140/200] loss=0.644792 avg=0.655561 VRAM=38.5GiB | 25.1% done | ETA(epoch): 799s + [Epoch 125][150/200] loss=0.607582 avg=0.655854 VRAM=38.6GiB | 25.1% done | ETA(epoch): 666s + [Epoch 125][160/200] loss=0.628392 avg=0.655298 VRAM=38.5GiB | 25.2% done | ETA(epoch): 533s + [Epoch 125][170/200] loss=0.743407 avg=0.654837 VRAM=38.6GiB | 25.2% done | ETA(epoch): 399s + [Epoch 125][180/200] loss=0.626623 avg=0.653654 VRAM=38.5GiB | 25.2% done | ETA(epoch): 266s + [Epoch 125][190/200] loss=0.586953 avg=0.655888 VRAM=38.6GiB | 25.2% done | ETA(epoch): 133s + [Epoch 125][200/200] loss=0.676446 avg=0.657457 VRAM=38.5GiB | 25.2% done | ETA(epoch): 0s + Train loss: 0.657457 (2662.4s) ETA: 18427min + Val loss: 0.679604 [t_0.0-0.2=0.9395 t_0.2-0.4=0.7560 t_0.4-0.6=0.6503 t_0.6-0.8=0.5528 t_0.8-1.0=0.5749] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 126/499 (25% done) --- + [Epoch 126][10/200] loss=0.683218 avg=0.668494 VRAM=38.6GiB | 25.2% done | ETA(epoch): 2525s + [Epoch 126][20/200] loss=0.622422 avg=0.673050 VRAM=38.5GiB | 25.2% done | ETA(epoch): 2391s + [Epoch 126][30/200] loss=0.529340 avg=0.672484 VRAM=38.6GiB | 25.2% done | ETA(epoch): 2258s + [Epoch 126][40/200] loss=0.545871 avg=0.673036 VRAM=38.5GiB | 25.2% done | ETA(epoch): 2137s + [Epoch 126][50/200] loss=0.576665 avg=0.672754 VRAM=38.6GiB | 25.2% done | ETA(epoch): 2002s + [Epoch 126][60/200] loss=0.547224 avg=0.666272 VRAM=38.5GiB | 25.3% done | ETA(epoch): 1866s + [Epoch 126][70/200] loss=0.615250 avg=0.668760 VRAM=38.6GiB | 25.3% done | ETA(epoch): 1733s + [Epoch 126][80/200] loss=0.588443 avg=0.666823 VRAM=38.5GiB | 25.3% done | ETA(epoch): 1598s + [Epoch 126][90/200] loss=0.651718 avg=0.669641 VRAM=38.6GiB | 25.3% done | ETA(epoch): 1465s + [Epoch 126][100/200] loss=0.800469 avg=0.671173 VRAM=38.5GiB | 25.3% done | ETA(epoch): 1331s + [Epoch 126][110/200] loss=0.646863 avg=0.668802 VRAM=38.6GiB | 25.3% done | ETA(epoch): 1198s + [Epoch 126][120/200] loss=0.482530 avg=0.666803 VRAM=38.5GiB | 25.3% done | ETA(epoch): 1065s + [Epoch 126][130/200] loss=0.791790 avg=0.665472 VRAM=38.6GiB | 25.3% done | ETA(epoch): 931s + [Epoch 126][140/200] loss=0.677722 avg=0.670575 VRAM=38.5GiB | 25.3% done | ETA(epoch): 798s + [Epoch 126][150/200] loss=0.660175 avg=0.669511 VRAM=38.6GiB | 25.4% done | ETA(epoch): 665s + [Epoch 126][160/200] loss=0.723306 avg=0.668948 VRAM=38.5GiB | 25.4% done | ETA(epoch): 532s + [Epoch 126][170/200] loss=0.529092 avg=0.667842 VRAM=38.6GiB | 25.4% done | ETA(epoch): 399s + [Epoch 126][180/200] loss=0.652244 avg=0.666897 VRAM=38.5GiB | 25.4% done | ETA(epoch): 266s + [Epoch 126][190/200] loss=0.612192 avg=0.667110 VRAM=38.6GiB | 25.4% done | ETA(epoch): 133s + [Epoch 126][200/200] loss=0.689640 avg=0.668055 VRAM=38.5GiB | 25.4% done | ETA(epoch): 0s + Train loss: 0.668055 (2659.0s) ETA: 18375min + Val loss: 0.690265 [t_0.0-0.2=0.9503 t_0.2-0.4=0.7464 t_0.4-0.6=0.6589 t_0.6-0.8=0.5584 t_0.8-1.0=0.5634] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 127/499 (25% done) --- + [Epoch 127][10/200] loss=0.707298 avg=0.626965 VRAM=38.6GiB | 25.4% done | ETA(epoch): 2522s + [Epoch 127][20/200] loss=0.699513 avg=0.656563 VRAM=38.5GiB | 25.4% done | ETA(epoch): 2389s + [Epoch 127][30/200] loss=0.582394 avg=0.648643 VRAM=38.6GiB | 25.4% done | ETA(epoch): 2257s + [Epoch 127][40/200] loss=0.757980 avg=0.652544 VRAM=38.5GiB | 25.4% done | ETA(epoch): 2125s + [Epoch 127][50/200] loss=0.612993 avg=0.645986 VRAM=38.6GiB | 25.4% done | ETA(epoch): 1992s + [Epoch 127][60/200] loss=0.634452 avg=0.650240 VRAM=38.5GiB | 25.5% done | ETA(epoch): 1859s + [Epoch 127][70/200] loss=0.765342 avg=0.653155 VRAM=38.6GiB | 25.5% done | ETA(epoch): 1727s + [Epoch 127][80/200] loss=0.694345 avg=0.657551 VRAM=38.5GiB | 25.5% done | ETA(epoch): 1594s + [Epoch 127][90/200] loss=0.551475 avg=0.656176 VRAM=38.6GiB | 25.5% done | ETA(epoch): 1461s + [Epoch 127][100/200] loss=0.671589 avg=0.656227 VRAM=38.5GiB | 25.5% done | ETA(epoch): 1333s + [Epoch 127][110/200] loss=0.577609 avg=0.654265 VRAM=38.6GiB | 25.5% done | ETA(epoch): 1199s + [Epoch 127][120/200] loss=0.705108 avg=0.652861 VRAM=38.5GiB | 25.5% done | ETA(epoch): 1065s + [Epoch 127][130/200] loss=0.632387 avg=0.651958 VRAM=38.6GiB | 25.5% done | ETA(epoch): 932s + [Epoch 127][140/200] loss=0.593266 avg=0.651037 VRAM=38.5GiB | 25.5% done | ETA(epoch): 799s + [Epoch 127][150/200] loss=0.653153 avg=0.651651 VRAM=38.6GiB | 25.6% done | ETA(epoch): 665s + [Epoch 127][160/200] loss=0.740089 avg=0.653411 VRAM=38.5GiB | 25.6% done | ETA(epoch): 532s + [Epoch 127][170/200] loss=0.932587 avg=0.654351 VRAM=38.6GiB | 25.6% done | ETA(epoch): 399s + [Epoch 127][180/200] loss=0.659888 avg=0.654901 VRAM=38.5GiB | 25.6% done | ETA(epoch): 266s + [Epoch 127][190/200] loss=0.737221 avg=0.656393 VRAM=38.6GiB | 25.6% done | ETA(epoch): 133s + [Epoch 127][200/200] loss=0.698603 avg=0.656516 VRAM=38.5GiB | 25.6% done | ETA(epoch): 0s + Train loss: 0.656516 (2663.1s) ETA: 18324min + Val loss: 0.678275 [t_0.0-0.2=0.9301 t_0.2-0.4=0.7642 t_0.4-0.6=0.6467 t_0.6-0.8=0.5482 t_0.8-1.0=0.5668] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 128/499 (26% done) --- + [Epoch 128][10/200] loss=0.628337 avg=0.667890 VRAM=38.6GiB | 25.6% done | ETA(epoch): 2520s + [Epoch 128][20/200] loss=0.686786 avg=0.662986 VRAM=38.5GiB | 25.6% done | ETA(epoch): 2392s + [Epoch 128][30/200] loss=0.631885 avg=0.643433 VRAM=38.6GiB | 25.6% done | ETA(epoch): 2258s + [Epoch 128][40/200] loss=0.601087 avg=0.636865 VRAM=38.5GiB | 25.6% done | ETA(epoch): 2126s + [Epoch 128][50/200] loss=0.621290 avg=0.636113 VRAM=38.6GiB | 25.7% done | ETA(epoch): 1992s + [Epoch 128][60/200] loss=0.654817 avg=0.636283 VRAM=38.5GiB | 25.7% done | ETA(epoch): 1871s + [Epoch 128][70/200] loss=0.715058 avg=0.635718 VRAM=38.6GiB | 25.7% done | ETA(epoch): 1736s + [Epoch 128][80/200] loss=0.659782 avg=0.637320 VRAM=38.5GiB | 25.7% done | ETA(epoch): 1601s + [Epoch 128][90/200] loss=0.642527 avg=0.638226 VRAM=38.6GiB | 25.7% done | ETA(epoch): 1467s + [Epoch 128][100/200] loss=0.581454 avg=0.642754 VRAM=38.5GiB | 25.7% done | ETA(epoch): 1333s + [Epoch 128][110/200] loss=0.647050 avg=0.643738 VRAM=38.6GiB | 25.7% done | ETA(epoch): 1200s + [Epoch 128][120/200] loss=0.572737 avg=0.645458 VRAM=38.5GiB | 25.7% done | ETA(epoch): 1066s + [Epoch 128][130/200] loss=0.713159 avg=0.651158 VRAM=38.6GiB | 25.7% done | ETA(epoch): 932s + [Epoch 128][140/200] loss=0.601001 avg=0.653183 VRAM=38.5GiB | 25.7% done | ETA(epoch): 799s + [Epoch 128][150/200] loss=0.707265 avg=0.652591 VRAM=38.6GiB | 25.8% done | ETA(epoch): 666s + [Epoch 128][160/200] loss=0.632850 avg=0.655094 VRAM=38.5GiB | 25.8% done | ETA(epoch): 533s + [Epoch 128][170/200] loss=0.570372 avg=0.654962 VRAM=38.6GiB | 25.8% done | ETA(epoch): 399s + [Epoch 128][180/200] loss=0.805752 avg=0.655055 VRAM=38.5GiB | 25.8% done | ETA(epoch): 266s + [Epoch 128][190/200] loss=0.593417 avg=0.654631 VRAM=38.6GiB | 25.8% done | ETA(epoch): 133s + [Epoch 128][200/200] loss=0.637922 avg=0.656462 VRAM=38.5GiB | 25.8% done | ETA(epoch): 0s + Train loss: 0.656462 (2661.7s) ETA: 18272min + Val loss: 0.701190 [t_0.0-0.2=0.9167 t_0.2-0.4=0.7474 t_0.4-0.6=0.6423 t_0.6-0.8=0.5623 t_0.8-1.0=0.5682] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 129/499 (26% done) --- + [Epoch 129][10/200] loss=0.775019 avg=0.681153 VRAM=38.6GiB | 25.8% done | ETA(epoch): 2521s + [Epoch 129][20/200] loss=0.577047 avg=0.645368 VRAM=38.5GiB | 25.8% done | ETA(epoch): 2429s + [Epoch 129][30/200] loss=0.585527 avg=0.651968 VRAM=38.6GiB | 25.8% done | ETA(epoch): 2282s + [Epoch 129][40/200] loss=0.653901 avg=0.645974 VRAM=38.5GiB | 25.8% done | ETA(epoch): 2142s + [Epoch 129][50/200] loss=0.551538 avg=0.645928 VRAM=38.6GiB | 25.9% done | ETA(epoch): 2005s + [Epoch 129][60/200] loss=0.624383 avg=0.646989 VRAM=38.5GiB | 25.9% done | ETA(epoch): 1869s + [Epoch 129][70/200] loss=0.651490 avg=0.655720 VRAM=38.6GiB | 25.9% done | ETA(epoch): 1734s + [Epoch 129][80/200] loss=0.594556 avg=0.653392 VRAM=38.5GiB | 25.9% done | ETA(epoch): 1600s + [Epoch 129][90/200] loss=0.717413 avg=0.651842 VRAM=38.6GiB | 25.9% done | ETA(epoch): 1466s + [Epoch 129][100/200] loss=0.661760 avg=0.653122 VRAM=38.5GiB | 25.9% done | ETA(epoch): 1332s + [Epoch 129][110/200] loss=0.630754 avg=0.652675 VRAM=38.6GiB | 25.9% done | ETA(epoch): 1201s + [Epoch 129][120/200] loss=0.748615 avg=0.650922 VRAM=38.5GiB | 25.9% done | ETA(epoch): 1067s + [Epoch 129][130/200] loss=0.711424 avg=0.649033 VRAM=38.6GiB | 25.9% done | ETA(epoch): 933s + [Epoch 129][140/200] loss=0.770317 avg=0.653392 VRAM=38.5GiB | 25.9% done | ETA(epoch): 800s + [Epoch 129][150/200] loss=0.523991 avg=0.652990 VRAM=38.6GiB | 25.9% done | ETA(epoch): 666s + [Epoch 129][160/200] loss=0.612890 avg=0.653139 VRAM=38.5GiB | 26.0% done | ETA(epoch): 533s + [Epoch 129][170/200] loss=0.696315 avg=0.651661 VRAM=38.6GiB | 26.0% done | ETA(epoch): 400s + [Epoch 129][180/200] loss=0.627076 avg=0.652484 VRAM=38.5GiB | 26.0% done | ETA(epoch): 266s + [Epoch 129][190/200] loss=0.710243 avg=0.653651 VRAM=38.6GiB | 26.0% done | ETA(epoch): 133s + [Epoch 129][200/200] loss=0.607817 avg=0.651759 VRAM=38.5GiB | 26.0% done | ETA(epoch): 0s + Train loss: 0.651759 (2663.0s) ETA: 18221min + Val loss: 0.698884 [t_0.0-0.2=0.9390 t_0.2-0.4=0.7471 t_0.4-0.6=0.6630 t_0.6-0.8=0.5666 t_0.8-1.0=0.5769] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 130/499 (26% done) --- + [Epoch 130][10/200] loss=0.649595 avg=0.606050 VRAM=38.6GiB | 26.0% done | ETA(epoch): 2521s + [Epoch 130][20/200] loss=0.573604 avg=0.614386 VRAM=38.5GiB | 26.0% done | ETA(epoch): 2392s + [Epoch 130][30/200] loss=0.745614 avg=0.635975 VRAM=38.6GiB | 26.0% done | ETA(epoch): 2258s + [Epoch 130][40/200] loss=0.631762 avg=0.635389 VRAM=38.5GiB | 26.0% done | ETA(epoch): 2125s + [Epoch 130][50/200] loss=0.592781 avg=0.638809 VRAM=38.6GiB | 26.1% done | ETA(epoch): 1992s + [Epoch 130][60/200] loss=0.605856 avg=0.645551 VRAM=38.5GiB | 26.1% done | ETA(epoch): 1860s + [Epoch 130][70/200] loss=0.625016 avg=0.652782 VRAM=38.6GiB | 26.1% done | ETA(epoch): 1742s + [Epoch 130][80/200] loss=0.818380 avg=0.658914 VRAM=38.5GiB | 26.1% done | ETA(epoch): 1606s + [Epoch 130][90/200] loss=0.750160 avg=0.664671 VRAM=38.6GiB | 26.1% done | ETA(epoch): 1471s + [Epoch 130][100/200] loss=0.600180 avg=0.662666 VRAM=38.5GiB | 26.1% done | ETA(epoch): 1336s + [Epoch 130][110/200] loss=0.541139 avg=0.660929 VRAM=38.6GiB | 26.1% done | ETA(epoch): 1202s + [Epoch 130][120/200] loss=0.664663 avg=0.658208 VRAM=38.5GiB | 26.1% done | ETA(epoch): 1068s + [Epoch 130][130/200] loss=0.668758 avg=0.658845 VRAM=38.6GiB | 26.1% done | ETA(epoch): 934s + [Epoch 130][140/200] loss=0.566242 avg=0.658130 VRAM=38.5GiB | 26.1% done | ETA(epoch): 800s + [Epoch 130][150/200] loss=0.670673 avg=0.656621 VRAM=38.6GiB | 26.2% done | ETA(epoch): 667s + [Epoch 130][160/200] loss=0.593036 avg=0.660004 VRAM=38.5GiB | 26.2% done | ETA(epoch): 533s + [Epoch 130][170/200] loss=0.599367 avg=0.660921 VRAM=38.6GiB | 26.2% done | ETA(epoch): 400s + [Epoch 130][180/200] loss=0.682772 avg=0.660598 VRAM=38.5GiB | 26.2% done | ETA(epoch): 267s + [Epoch 130][190/200] loss=0.660062 avg=0.660095 VRAM=38.6GiB | 26.2% done | ETA(epoch): 133s + [Epoch 130][200/200] loss=0.618269 avg=0.659179 VRAM=38.5GiB | 26.2% done | ETA(epoch): 0s + Train loss: 0.659179 (2664.7s) ETA: 18170min + Val loss: 0.708081 [t_0.0-0.2=0.9357 t_0.2-0.4=0.7666 t_0.4-0.6=0.6433 t_0.6-0.8=0.5486 t_0.8-1.0=0.5717] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 131/499 (26% done) --- + [Epoch 131][10/200] loss=0.537397 avg=0.646063 VRAM=38.6GiB | 26.2% done | ETA(epoch): 2523s + [Epoch 131][20/200] loss=0.607923 avg=0.644293 VRAM=38.5GiB | 26.2% done | ETA(epoch): 2392s + [Epoch 131][30/200] loss=0.669796 avg=0.645224 VRAM=38.6GiB | 26.2% done | ETA(epoch): 2259s + [Epoch 131][40/200] loss=0.643449 avg=0.640367 VRAM=38.5GiB | 26.2% done | ETA(epoch): 2136s + [Epoch 131][50/200] loss=0.559270 avg=0.643524 VRAM=38.6GiB | 26.2% done | ETA(epoch): 2001s + [Epoch 131][60/200] loss=0.564036 avg=0.645426 VRAM=38.5GiB | 26.3% done | ETA(epoch): 1866s + [Epoch 131][70/200] loss=0.599134 avg=0.641617 VRAM=38.6GiB | 26.3% done | ETA(epoch): 1732s + [Epoch 131][80/200] loss=0.672531 avg=0.643546 VRAM=38.5GiB | 26.3% done | ETA(epoch): 1598s + [Epoch 131][90/200] loss=0.587944 avg=0.641119 VRAM=38.6GiB | 26.3% done | ETA(epoch): 1465s + [Epoch 131][100/200] loss=0.591201 avg=0.640595 VRAM=38.5GiB | 26.3% done | ETA(epoch): 1331s + [Epoch 131][110/200] loss=0.712622 avg=0.645074 VRAM=38.6GiB | 26.3% done | ETA(epoch): 1198s + [Epoch 131][120/200] loss=0.601909 avg=0.646010 VRAM=38.5GiB | 26.3% done | ETA(epoch): 1064s + [Epoch 131][130/200] loss=0.670346 avg=0.647992 VRAM=38.6GiB | 26.3% done | ETA(epoch): 933s + [Epoch 131][140/200] loss=0.716375 avg=0.649510 VRAM=38.5GiB | 26.3% done | ETA(epoch): 800s + [Epoch 131][150/200] loss=0.721986 avg=0.651262 VRAM=38.6GiB | 26.4% done | ETA(epoch): 666s + [Epoch 131][160/200] loss=0.679334 avg=0.653332 VRAM=38.5GiB | 26.4% done | ETA(epoch): 533s + [Epoch 131][170/200] loss=0.679493 avg=0.650698 VRAM=38.6GiB | 26.4% done | ETA(epoch): 400s + [Epoch 131][180/200] loss=0.703656 avg=0.653731 VRAM=38.5GiB | 26.4% done | ETA(epoch): 266s + [Epoch 131][190/200] loss=0.638737 avg=0.651773 VRAM=38.6GiB | 26.4% done | ETA(epoch): 133s + [Epoch 131][200/200] loss=0.642782 avg=0.651701 VRAM=38.5GiB | 26.4% done | ETA(epoch): 0s + Train loss: 0.651701 (2663.7s) ETA: 18119min + Val loss: 0.689533 [t_0.0-0.2=0.9264 t_0.2-0.4=0.7431 t_0.4-0.6=0.6290 t_0.6-0.8=0.5601 t_0.8-1.0=0.5729] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 132/499 (26% done) --- + [Epoch 132][10/200] loss=0.605912 avg=0.695501 VRAM=38.6GiB | 26.4% done | ETA(epoch): 2521s + [Epoch 132][20/200] loss=0.621429 avg=0.682812 VRAM=38.5GiB | 26.4% done | ETA(epoch): 2391s + [Epoch 132][30/200] loss=0.773721 avg=0.680893 VRAM=38.6GiB | 26.4% done | ETA(epoch): 2258s + [Epoch 132][40/200] loss=0.654624 avg=0.682252 VRAM=38.5GiB | 26.4% done | ETA(epoch): 2126s + [Epoch 132][50/200] loss=0.603287 avg=0.669797 VRAM=38.6GiB | 26.5% done | ETA(epoch): 1992s + [Epoch 132][60/200] loss=0.596378 avg=0.672007 VRAM=38.5GiB | 26.5% done | ETA(epoch): 1861s + [Epoch 132][70/200] loss=0.687801 avg=0.665613 VRAM=38.6GiB | 26.5% done | ETA(epoch): 1727s + [Epoch 132][80/200] loss=0.742168 avg=0.665525 VRAM=38.5GiB | 26.5% done | ETA(epoch): 1594s + [Epoch 132][90/200] loss=0.565650 avg=0.667802 VRAM=38.6GiB | 26.5% done | ETA(epoch): 1465s + [Epoch 132][100/200] loss=0.750370 avg=0.672302 VRAM=38.5GiB | 26.5% done | ETA(epoch): 1331s + [Epoch 132][110/200] loss=0.687333 avg=0.670349 VRAM=38.6GiB | 26.5% done | ETA(epoch): 1198s + [Epoch 132][120/200] loss=0.502583 avg=0.669930 VRAM=38.5GiB | 26.5% done | ETA(epoch): 1064s + [Epoch 132][130/200] loss=0.588706 avg=0.667169 VRAM=38.6GiB | 26.5% done | ETA(epoch): 931s + [Epoch 132][140/200] loss=0.585467 avg=0.665959 VRAM=38.5GiB | 26.5% done | ETA(epoch): 798s + [Epoch 132][150/200] loss=0.674809 avg=0.663541 VRAM=38.6GiB | 26.6% done | ETA(epoch): 665s + [Epoch 132][160/200] loss=0.760898 avg=0.664714 VRAM=38.5GiB | 26.6% done | ETA(epoch): 532s + [Epoch 132][170/200] loss=0.684080 avg=0.666391 VRAM=38.6GiB | 26.6% done | ETA(epoch): 399s + [Epoch 132][180/200] loss=0.604063 avg=0.663948 VRAM=38.5GiB | 26.6% done | ETA(epoch): 266s + [Epoch 132][190/200] loss=0.738564 avg=0.664120 VRAM=38.6GiB | 26.6% done | ETA(epoch): 133s + [Epoch 132][200/200] loss=0.545334 avg=0.663911 VRAM=38.5GiB | 26.6% done | ETA(epoch): 0s + Train loss: 0.663911 (2660.1s) ETA: 18068min + Val loss: 0.675671 [t_0.0-0.2=0.9416 t_0.2-0.4=0.7583 t_0.4-0.6=0.6398 t_0.6-0.8=0.5630 t_0.8-1.0=0.5508] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 133/499 (27% done) --- + [Epoch 133][10/200] loss=0.649333 avg=0.669026 VRAM=38.6GiB | 26.6% done | ETA(epoch): 2518s + [Epoch 133][20/200] loss=0.642121 avg=0.671054 VRAM=38.5GiB | 26.6% done | ETA(epoch): 2390s + [Epoch 133][30/200] loss=0.621790 avg=0.670704 VRAM=38.6GiB | 26.6% done | ETA(epoch): 2256s + [Epoch 133][40/200] loss=0.836065 avg=0.668643 VRAM=38.5GiB | 26.6% done | ETA(epoch): 2124s + [Epoch 133][50/200] loss=0.707278 avg=0.672275 VRAM=38.6GiB | 26.7% done | ETA(epoch): 1991s + [Epoch 133][60/200] loss=0.638739 avg=0.667928 VRAM=38.5GiB | 26.7% done | ETA(epoch): 1858s + [Epoch 133][70/200] loss=0.681808 avg=0.667797 VRAM=38.6GiB | 26.7% done | ETA(epoch): 1726s + [Epoch 133][80/200] loss=0.822542 avg=0.670710 VRAM=38.5GiB | 26.7% done | ETA(epoch): 1593s + [Epoch 133][90/200] loss=0.599262 avg=0.666805 VRAM=38.6GiB | 26.7% done | ETA(epoch): 1460s + [Epoch 133][100/200] loss=0.831930 avg=0.670670 VRAM=38.5GiB | 26.7% done | ETA(epoch): 1328s + [Epoch 133][110/200] loss=0.661454 avg=0.670372 VRAM=38.6GiB | 26.7% done | ETA(epoch): 1195s + [Epoch 133][120/200] loss=0.699480 avg=0.670010 VRAM=38.5GiB | 26.7% done | ETA(epoch): 1062s + [Epoch 133][130/200] loss=0.709056 avg=0.669702 VRAM=38.6GiB | 26.7% done | ETA(epoch): 929s + [Epoch 133][140/200] loss=0.588570 avg=0.665505 VRAM=38.5GiB | 26.7% done | ETA(epoch): 797s + [Epoch 133][150/200] loss=0.798005 avg=0.667126 VRAM=38.6GiB | 26.8% done | ETA(epoch): 664s + [Epoch 133][160/200] loss=0.723734 avg=0.668103 VRAM=38.5GiB | 26.8% done | ETA(epoch): 531s + [Epoch 133][170/200] loss=0.666406 avg=0.666780 VRAM=38.6GiB | 26.8% done | ETA(epoch): 398s + [Epoch 133][180/200] loss=0.667461 avg=0.666502 VRAM=38.5GiB | 26.8% done | ETA(epoch): 266s + [Epoch 133][190/200] loss=0.632083 avg=0.667270 VRAM=38.6GiB | 26.8% done | ETA(epoch): 133s + [Epoch 133][200/200] loss=0.620585 avg=0.667237 VRAM=38.5GiB | 26.8% done | ETA(epoch): 0s + Train loss: 0.667237 (2656.0s) ETA: 18016min + Val loss: 0.692247 [t_0.0-0.2=0.9172 t_0.2-0.4=0.7683 t_0.4-0.6=0.6500 t_0.6-0.8=0.5490 t_0.8-1.0=0.5723] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 134/499 (27% done) --- + [Epoch 134][10/200] loss=0.707661 avg=0.650634 VRAM=38.6GiB | 26.8% done | ETA(epoch): 2518s + [Epoch 134][20/200] loss=0.696467 avg=0.652700 VRAM=38.5GiB | 26.8% done | ETA(epoch): 2390s + [Epoch 134][30/200] loss=0.690513 avg=0.667184 VRAM=38.6GiB | 26.8% done | ETA(epoch): 2257s + [Epoch 134][40/200] loss=0.604910 avg=0.667393 VRAM=38.5GiB | 26.8% done | ETA(epoch): 2124s + [Epoch 134][50/200] loss=0.678755 avg=0.658878 VRAM=38.6GiB | 26.9% done | ETA(epoch): 1993s + [Epoch 134][60/200] loss=0.593245 avg=0.660243 VRAM=38.5GiB | 26.9% done | ETA(epoch): 1859s + [Epoch 134][70/200] loss=0.721068 avg=0.657138 VRAM=38.6GiB | 26.9% done | ETA(epoch): 1727s + [Epoch 134][80/200] loss=0.794108 avg=0.661499 VRAM=38.5GiB | 26.9% done | ETA(epoch): 1594s + [Epoch 134][90/200] loss=0.628514 avg=0.658802 VRAM=38.6GiB | 26.9% done | ETA(epoch): 1461s + [Epoch 134][100/200] loss=0.822697 avg=0.659132 VRAM=38.5GiB | 26.9% done | ETA(epoch): 1328s + [Epoch 134][110/200] loss=0.518786 avg=0.658441 VRAM=38.6GiB | 26.9% done | ETA(epoch): 1195s + [Epoch 134][120/200] loss=0.683698 avg=0.658782 VRAM=38.5GiB | 26.9% done | ETA(epoch): 1062s + [Epoch 134][130/200] loss=0.667685 avg=0.660279 VRAM=38.6GiB | 26.9% done | ETA(epoch): 930s + [Epoch 134][140/200] loss=0.695994 avg=0.660247 VRAM=38.5GiB | 26.9% done | ETA(epoch): 797s + [Epoch 134][150/200] loss=0.670376 avg=0.656583 VRAM=38.6GiB | 27.0% done | ETA(epoch): 664s + [Epoch 134][160/200] loss=0.587898 avg=0.657047 VRAM=38.5GiB | 27.0% done | ETA(epoch): 531s + [Epoch 134][170/200] loss=0.649149 avg=0.657814 VRAM=38.6GiB | 27.0% done | ETA(epoch): 398s + [Epoch 134][180/200] loss=0.689427 avg=0.659051 VRAM=38.5GiB | 27.0% done | ETA(epoch): 266s + [Epoch 134][190/200] loss=0.614021 avg=0.659350 VRAM=38.6GiB | 27.0% done | ETA(epoch): 133s + [Epoch 134][200/200] loss=0.550810 avg=0.658443 VRAM=38.5GiB | 27.0% done | ETA(epoch): 0s + Train loss: 0.658443 (2658.4s) ETA: 17965min + Val loss: 0.713743 [t_0.0-0.2=0.9677 t_0.2-0.4=0.7586 t_0.4-0.6=0.6330 t_0.6-0.8=0.5655 t_0.8-1.0=0.5756] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 135/499 (27% done) --- + [Epoch 135][10/200] loss=0.670702 avg=0.662879 VRAM=38.6GiB | 27.0% done | ETA(epoch): 2519s + [Epoch 135][20/200] loss=0.562721 avg=0.636546 VRAM=38.5GiB | 27.0% done | ETA(epoch): 2389s + [Epoch 135][30/200] loss=0.570599 avg=0.637463 VRAM=38.6GiB | 27.0% done | ETA(epoch): 2256s + [Epoch 135][40/200] loss=0.722146 avg=0.639356 VRAM=38.5GiB | 27.0% done | ETA(epoch): 2124s + [Epoch 135][50/200] loss=0.608796 avg=0.640887 VRAM=38.6GiB | 27.1% done | ETA(epoch): 1991s + [Epoch 135][60/200] loss=0.800964 avg=0.648668 VRAM=38.5GiB | 27.1% done | ETA(epoch): 1859s + [Epoch 135][70/200] loss=0.631617 avg=0.651338 VRAM=38.6GiB | 27.1% done | ETA(epoch): 1732s + [Epoch 135][80/200] loss=0.638676 avg=0.651345 VRAM=38.5GiB | 27.1% done | ETA(epoch): 1597s + [Epoch 135][90/200] loss=0.601516 avg=0.646828 VRAM=38.6GiB | 27.1% done | ETA(epoch): 1464s + [Epoch 135][100/200] loss=0.656142 avg=0.646050 VRAM=38.5GiB | 27.1% done | ETA(epoch): 1330s + [Epoch 135][110/200] loss=0.608056 avg=0.646850 VRAM=38.6GiB | 27.1% done | ETA(epoch): 1197s + [Epoch 135][120/200] loss=0.721898 avg=0.646155 VRAM=38.5GiB | 27.1% done | ETA(epoch): 1064s + [Epoch 135][130/200] loss=0.688358 avg=0.650259 VRAM=38.6GiB | 27.1% done | ETA(epoch): 931s + [Epoch 135][140/200] loss=0.789312 avg=0.653988 VRAM=38.5GiB | 27.1% done | ETA(epoch): 798s + [Epoch 135][150/200] loss=0.636151 avg=0.654418 VRAM=38.6GiB | 27.2% done | ETA(epoch): 665s + [Epoch 135][160/200] loss=0.754428 avg=0.655151 VRAM=38.5GiB | 27.2% done | ETA(epoch): 532s + [Epoch 135][170/200] loss=0.691129 avg=0.659712 VRAM=38.6GiB | 27.2% done | ETA(epoch): 399s + [Epoch 135][180/200] loss=0.597653 avg=0.658730 VRAM=38.5GiB | 27.2% done | ETA(epoch): 266s + [Epoch 135][190/200] loss=0.746053 avg=0.661962 VRAM=38.6GiB | 27.2% done | ETA(epoch): 133s + [Epoch 135][200/200] loss=0.627880 avg=0.661458 VRAM=38.5GiB | 27.2% done | ETA(epoch): 0s + Train loss: 0.661458 (2658.6s) ETA: 17914min + Val loss: 0.689155 [t_0.0-0.2=0.9303 t_0.2-0.4=0.7713 t_0.4-0.6=0.6472 t_0.6-0.8=0.5561 t_0.8-1.0=0.5668] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 136/499 (27% done) --- + [Epoch 136][10/200] loss=0.523329 avg=0.653722 VRAM=38.6GiB | 27.2% done | ETA(epoch): 2521s + [Epoch 136][20/200] loss=0.644074 avg=0.665783 VRAM=38.5GiB | 27.2% done | ETA(epoch): 2389s + [Epoch 136][30/200] loss=0.607727 avg=0.654122 VRAM=38.6GiB | 27.2% done | ETA(epoch): 2278s + [Epoch 136][40/200] loss=0.735750 avg=0.648617 VRAM=38.5GiB | 27.2% done | ETA(epoch): 2139s + [Epoch 136][50/200] loss=0.710706 avg=0.655041 VRAM=38.6GiB | 27.3% done | ETA(epoch): 2003s + [Epoch 136][60/200] loss=0.684441 avg=0.658493 VRAM=38.5GiB | 27.3% done | ETA(epoch): 1868s + [Epoch 136][70/200] loss=0.594718 avg=0.661053 VRAM=38.6GiB | 27.3% done | ETA(epoch): 1733s + [Epoch 136][80/200] loss=0.622153 avg=0.666760 VRAM=38.5GiB | 27.3% done | ETA(epoch): 1599s + [Epoch 136][90/200] loss=0.779459 avg=0.664992 VRAM=38.6GiB | 27.3% done | ETA(epoch): 1465s + [Epoch 136][100/200] loss=0.631485 avg=0.665455 VRAM=38.5GiB | 27.3% done | ETA(epoch): 1331s + [Epoch 136][110/200] loss=0.655181 avg=0.664752 VRAM=38.6GiB | 27.3% done | ETA(epoch): 1198s + [Epoch 136][120/200] loss=0.558963 avg=0.664826 VRAM=38.5GiB | 27.3% done | ETA(epoch): 1066s + [Epoch 136][130/200] loss=0.559169 avg=0.665877 VRAM=38.6GiB | 27.3% done | ETA(epoch): 933s + [Epoch 136][140/200] loss=0.592248 avg=0.663393 VRAM=38.5GiB | 27.3% done | ETA(epoch): 800s + [Epoch 136][150/200] loss=0.700193 avg=0.664772 VRAM=38.6GiB | 27.4% done | ETA(epoch): 666s + [Epoch 136][160/200] loss=0.625301 avg=0.666903 VRAM=38.5GiB | 27.4% done | ETA(epoch): 533s + [Epoch 136][170/200] loss=0.716172 avg=0.665516 VRAM=38.6GiB | 27.4% done | ETA(epoch): 400s + [Epoch 136][180/200] loss=0.686331 avg=0.666082 VRAM=38.5GiB | 27.4% done | ETA(epoch): 266s + [Epoch 136][190/200] loss=0.808863 avg=0.666722 VRAM=38.6GiB | 27.4% done | ETA(epoch): 133s + [Epoch 136][200/200] loss=0.695237 avg=0.666157 VRAM=38.5GiB | 27.4% done | ETA(epoch): 0s + Train loss: 0.666157 (2662.1s) ETA: 17863min + Val loss: 0.659657 [t_0.0-0.2=0.9193 t_0.2-0.4=0.7534 t_0.4-0.6=0.6421 t_0.6-0.8=0.5626 t_0.8-1.0=0.5627] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0136 (BEST) + Deleted old checkpoint: checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 137/499 (27% done) --- + [Epoch 137][10/200] loss=0.646087 avg=0.676789 VRAM=38.6GiB | 27.4% done | ETA(epoch): 2522s + [Epoch 137][20/200] loss=0.698708 avg=0.687628 VRAM=38.5GiB | 27.4% done | ETA(epoch): 2391s + [Epoch 137][30/200] loss=0.843761 avg=0.674408 VRAM=38.6GiB | 27.4% done | ETA(epoch): 2258s + [Epoch 137][40/200] loss=0.666962 avg=0.673739 VRAM=38.5GiB | 27.4% done | ETA(epoch): 2124s + [Epoch 137][50/200] loss=0.628119 avg=0.664534 VRAM=38.6GiB | 27.5% done | ETA(epoch): 1991s + [Epoch 137][60/200] loss=0.608708 avg=0.663047 VRAM=38.5GiB | 27.5% done | ETA(epoch): 1859s + [Epoch 137][70/200] loss=0.615291 avg=0.660947 VRAM=38.6GiB | 27.5% done | ETA(epoch): 1726s + [Epoch 137][80/200] loss=0.586550 avg=0.660941 VRAM=38.5GiB | 27.5% done | ETA(epoch): 1604s + [Epoch 137][90/200] loss=0.613759 avg=0.663000 VRAM=38.6GiB | 27.5% done | ETA(epoch): 1469s + [Epoch 137][100/200] loss=0.632698 avg=0.664531 VRAM=38.5GiB | 27.5% done | ETA(epoch): 1335s + [Epoch 137][110/200] loss=0.730810 avg=0.666782 VRAM=38.6GiB | 27.5% done | ETA(epoch): 1201s + [Epoch 137][120/200] loss=0.704780 avg=0.664689 VRAM=38.5GiB | 27.5% done | ETA(epoch): 1067s + [Epoch 137][130/200] loss=0.572599 avg=0.665371 VRAM=38.6GiB | 27.5% done | ETA(epoch): 933s + [Epoch 137][140/200] loss=0.594569 avg=0.664535 VRAM=38.5GiB | 27.5% done | ETA(epoch): 800s + [Epoch 137][150/200] loss=0.770362 avg=0.666817 VRAM=38.6GiB | 27.6% done | ETA(epoch): 666s + [Epoch 137][160/200] loss=0.618069 avg=0.666126 VRAM=38.5GiB | 27.6% done | ETA(epoch): 533s + [Epoch 137][170/200] loss=0.679633 avg=0.665868 VRAM=38.6GiB | 27.6% done | ETA(epoch): 400s + [Epoch 137][180/200] loss=0.626914 avg=0.668731 VRAM=38.5GiB | 27.6% done | ETA(epoch): 267s + [Epoch 137][190/200] loss=0.741287 avg=0.667537 VRAM=38.6GiB | 27.6% done | ETA(epoch): 133s + [Epoch 137][200/200] loss=0.748187 avg=0.666891 VRAM=38.5GiB | 27.6% done | ETA(epoch): 0s + Train loss: 0.666891 (2665.6s) ETA: 17812min + Val loss: 0.697597 [t_0.0-0.2=0.9458 t_0.2-0.4=0.7413 t_0.4-0.6=0.6427 t_0.6-0.8=0.5630 t_0.8-1.0=0.5595] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 138/499 (28% done) --- + [Epoch 138][10/200] loss=0.681861 avg=0.656935 VRAM=38.6GiB | 27.6% done | ETA(epoch): 2523s + [Epoch 138][20/200] loss=0.656597 avg=0.649902 VRAM=38.5GiB | 27.6% done | ETA(epoch): 2388s + [Epoch 138][30/200] loss=0.698429 avg=0.663983 VRAM=38.6GiB | 27.6% done | ETA(epoch): 2256s + [Epoch 138][40/200] loss=0.717072 avg=0.674380 VRAM=38.5GiB | 27.6% done | ETA(epoch): 2123s + [Epoch 138][50/200] loss=0.651310 avg=0.675409 VRAM=38.6GiB | 27.7% done | ETA(epoch): 1999s + [Epoch 138][60/200] loss=0.636829 avg=0.669998 VRAM=38.5GiB | 27.7% done | ETA(epoch): 1864s + [Epoch 138][70/200] loss=0.637884 avg=0.672341 VRAM=38.6GiB | 27.7% done | ETA(epoch): 1731s + [Epoch 138][80/200] loss=0.822001 avg=0.669497 VRAM=38.5GiB | 27.7% done | ETA(epoch): 1597s + [Epoch 138][90/200] loss=0.661732 avg=0.670287 VRAM=38.6GiB | 27.7% done | ETA(epoch): 1464s + [Epoch 138][100/200] loss=0.584027 avg=0.670393 VRAM=38.5GiB | 27.7% done | ETA(epoch): 1330s + [Epoch 138][110/200] loss=0.597763 avg=0.671190 VRAM=38.6GiB | 27.7% done | ETA(epoch): 1197s + [Epoch 138][120/200] loss=0.596038 avg=0.668139 VRAM=38.5GiB | 27.7% done | ETA(epoch): 1064s + [Epoch 138][130/200] loss=0.731828 avg=0.668200 VRAM=38.6GiB | 27.7% done | ETA(epoch): 931s + [Epoch 138][140/200] loss=0.730880 avg=0.666699 VRAM=38.5GiB | 27.7% done | ETA(epoch): 799s + [Epoch 138][150/200] loss=0.494198 avg=0.665464 VRAM=38.6GiB | 27.8% done | ETA(epoch): 666s + [Epoch 138][160/200] loss=0.912637 avg=0.665137 VRAM=38.5GiB | 27.8% done | ETA(epoch): 533s + [Epoch 138][170/200] loss=0.704078 avg=0.663651 VRAM=38.6GiB | 27.8% done | ETA(epoch): 399s + [Epoch 138][180/200] loss=0.575369 avg=0.664386 VRAM=38.5GiB | 27.8% done | ETA(epoch): 266s + [Epoch 138][190/200] loss=0.577033 avg=0.663642 VRAM=38.6GiB | 27.8% done | ETA(epoch): 133s + [Epoch 138][200/200] loss=0.610337 avg=0.662987 VRAM=38.5GiB | 27.8% done | ETA(epoch): 0s + Train loss: 0.662987 (2661.1s) ETA: 17761min + Val loss: 0.678857 [t_0.0-0.2=0.9143 t_0.2-0.4=0.7478 t_0.4-0.6=0.6384 t_0.6-0.8=0.5773 t_0.8-1.0=0.5623] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 139/499 (28% done) --- + [Epoch 139][10/200] loss=0.607525 avg=0.662263 VRAM=38.6GiB | 27.8% done | ETA(epoch): 2544s + [Epoch 139][20/200] loss=0.681538 avg=0.661165 VRAM=38.5GiB | 27.8% done | ETA(epoch): 2399s + [Epoch 139][30/200] loss=0.629059 avg=0.643378 VRAM=38.6GiB | 27.8% done | ETA(epoch): 2263s + [Epoch 139][40/200] loss=0.689322 avg=0.635547 VRAM=38.5GiB | 27.8% done | ETA(epoch): 2128s + [Epoch 139][50/200] loss=0.630855 avg=0.648212 VRAM=38.6GiB | 27.9% done | ETA(epoch): 1995s + [Epoch 139][60/200] loss=0.682388 avg=0.652858 VRAM=38.5GiB | 27.9% done | ETA(epoch): 1862s + [Epoch 139][70/200] loss=0.656995 avg=0.656345 VRAM=38.6GiB | 27.9% done | ETA(epoch): 1728s + [Epoch 139][80/200] loss=0.833588 avg=0.655581 VRAM=38.5GiB | 27.9% done | ETA(epoch): 1595s + [Epoch 139][90/200] loss=0.557076 avg=0.655262 VRAM=38.6GiB | 27.9% done | ETA(epoch): 1462s + [Epoch 139][100/200] loss=0.609723 avg=0.656346 VRAM=38.5GiB | 27.9% done | ETA(epoch): 1332s + [Epoch 139][110/200] loss=0.580160 avg=0.656863 VRAM=38.6GiB | 27.9% done | ETA(epoch): 1198s + [Epoch 139][120/200] loss=0.596816 avg=0.655464 VRAM=38.5GiB | 27.9% done | ETA(epoch): 1065s + [Epoch 139][130/200] loss=0.616695 avg=0.656662 VRAM=38.6GiB | 27.9% done | ETA(epoch): 932s + [Epoch 139][140/200] loss=0.688985 avg=0.656700 VRAM=38.5GiB | 27.9% done | ETA(epoch): 798s + [Epoch 139][150/200] loss=0.670808 avg=0.656460 VRAM=38.6GiB | 28.0% done | ETA(epoch): 665s + [Epoch 139][160/200] loss=0.644472 avg=0.654276 VRAM=38.5GiB | 28.0% done | ETA(epoch): 532s + [Epoch 139][170/200] loss=0.703917 avg=0.654933 VRAM=38.6GiB | 28.0% done | ETA(epoch): 399s + [Epoch 139][180/200] loss=0.578813 avg=0.654195 VRAM=38.5GiB | 28.0% done | ETA(epoch): 266s + [Epoch 139][190/200] loss=0.684042 avg=0.654099 VRAM=38.6GiB | 28.0% done | ETA(epoch): 133s + [Epoch 139][200/200] loss=0.655564 avg=0.655744 VRAM=38.5GiB | 28.0% done | ETA(epoch): 0s + Train loss: 0.655744 (2659.9s) ETA: 17710min + Val loss: 0.687117 [t_0.0-0.2=0.9512 t_0.2-0.4=0.7702 t_0.4-0.6=0.6444 t_0.6-0.8=0.5557 t_0.8-1.0=0.5573] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0139 +[MEM @ epoch 139 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 140/499 (28% done) --- + [Epoch 140][10/200] loss=0.600513 avg=0.695794 VRAM=38.6GiB | 28.0% done | ETA(epoch): 2522s + [Epoch 140][20/200] loss=0.599970 avg=0.672556 VRAM=38.5GiB | 28.0% done | ETA(epoch): 2388s + [Epoch 140][30/200] loss=0.724903 avg=0.676698 VRAM=38.6GiB | 28.0% done | ETA(epoch): 2256s + [Epoch 140][40/200] loss=0.801167 avg=0.667545 VRAM=38.5GiB | 28.0% done | ETA(epoch): 2123s + [Epoch 140][50/200] loss=0.613985 avg=0.663963 VRAM=38.6GiB | 28.1% done | ETA(epoch): 1991s + [Epoch 140][60/200] loss=0.634552 avg=0.665524 VRAM=38.5GiB | 28.1% done | ETA(epoch): 1858s + [Epoch 140][70/200] loss=0.520510 avg=0.660941 VRAM=38.6GiB | 28.1% done | ETA(epoch): 1725s + [Epoch 140][80/200] loss=0.575730 avg=0.661370 VRAM=38.5GiB | 28.1% done | ETA(epoch): 1592s + [Epoch 140][90/200] loss=0.745516 avg=0.662778 VRAM=38.6GiB | 28.1% done | ETA(epoch): 1459s + [Epoch 140][100/200] loss=0.683836 avg=0.665070 VRAM=38.5GiB | 28.1% done | ETA(epoch): 1326s + [Epoch 140][110/200] loss=0.638295 avg=0.665508 VRAM=38.6GiB | 28.1% done | ETA(epoch): 1194s + [Epoch 140][120/200] loss=0.671896 avg=0.668371 VRAM=38.5GiB | 28.1% done | ETA(epoch): 1061s + [Epoch 140][130/200] loss=0.755374 avg=0.666692 VRAM=38.6GiB | 28.1% done | ETA(epoch): 928s + [Epoch 140][140/200] loss=0.617104 avg=0.666927 VRAM=38.5GiB | 28.1% done | ETA(epoch): 796s + [Epoch 140][150/200] loss=0.517460 avg=0.666037 VRAM=38.6GiB | 28.1% done | ETA(epoch): 663s + [Epoch 140][160/200] loss=0.604378 avg=0.665592 VRAM=38.5GiB | 28.2% done | ETA(epoch): 531s + [Epoch 140][170/200] loss=0.607366 avg=0.667380 VRAM=38.6GiB | 28.2% done | ETA(epoch): 398s + [Epoch 140][180/200] loss=0.657185 avg=0.665713 VRAM=38.5GiB | 28.2% done | ETA(epoch): 265s + [Epoch 140][190/200] loss=0.522297 avg=0.666769 VRAM=38.6GiB | 28.2% done | ETA(epoch): 133s + [Epoch 140][200/200] loss=0.506149 avg=0.664737 VRAM=38.5GiB | 28.2% done | ETA(epoch): 0s + Train loss: 0.664737 (2652.7s) ETA: 17659min + Val loss: 0.698626 [t_0.0-0.2=0.9610 t_0.2-0.4=0.7494 t_0.4-0.6=0.6569 t_0.6-0.8=0.5644 t_0.8-1.0=0.5790] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 23.0/188.5 GiB (12.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 141/499 (28% done) --- + [Epoch 141][10/200] loss=0.643399 avg=0.667840 VRAM=38.6GiB | 28.2% done | ETA(epoch): 2517s + [Epoch 141][20/200] loss=0.722356 avg=0.666543 VRAM=38.5GiB | 28.2% done | ETA(epoch): 2384s + [Epoch 141][30/200] loss=0.541811 avg=0.658460 VRAM=38.6GiB | 28.2% done | ETA(epoch): 2253s + [Epoch 141][40/200] loss=0.542467 avg=0.648576 VRAM=38.5GiB | 28.2% done | ETA(epoch): 2120s + [Epoch 141][50/200] loss=0.667122 avg=0.650088 VRAM=38.6GiB | 28.2% done | ETA(epoch): 1987s + [Epoch 141][60/200] loss=0.572317 avg=0.654695 VRAM=38.5GiB | 28.3% done | ETA(epoch): 1856s + [Epoch 141][70/200] loss=0.632075 avg=0.650528 VRAM=38.6GiB | 28.3% done | ETA(epoch): 1723s + [Epoch 141][80/200] loss=0.718256 avg=0.652372 VRAM=38.5GiB | 28.3% done | ETA(epoch): 1591s + [Epoch 141][90/200] loss=0.601507 avg=0.653912 VRAM=38.6GiB | 28.3% done | ETA(epoch): 1458s + [Epoch 141][100/200] loss=0.613343 avg=0.654537 VRAM=38.5GiB | 28.3% done | ETA(epoch): 1326s + [Epoch 141][110/200] loss=0.758486 avg=0.656190 VRAM=38.6GiB | 28.3% done | ETA(epoch): 1193s + [Epoch 141][120/200] loss=0.737638 avg=0.654960 VRAM=38.5GiB | 28.3% done | ETA(epoch): 1061s + [Epoch 141][130/200] loss=0.667375 avg=0.654091 VRAM=38.6GiB | 28.3% done | ETA(epoch): 928s + [Epoch 141][140/200] loss=0.692002 avg=0.656223 VRAM=38.5GiB | 28.3% done | ETA(epoch): 795s + [Epoch 141][150/200] loss=0.781207 avg=0.656322 VRAM=38.6GiB | 28.3% done | ETA(epoch): 663s + [Epoch 141][160/200] loss=0.696451 avg=0.657610 VRAM=38.5GiB | 28.4% done | ETA(epoch): 530s + [Epoch 141][170/200] loss=0.651684 avg=0.658178 VRAM=38.6GiB | 28.4% done | ETA(epoch): 398s + [Epoch 141][180/200] loss=0.631400 avg=0.659373 VRAM=38.5GiB | 28.4% done | ETA(epoch): 265s + [Epoch 141][190/200] loss=0.632966 avg=0.659319 VRAM=38.6GiB | 28.4% done | ETA(epoch): 133s + [Epoch 141][200/200] loss=0.591429 avg=0.657668 VRAM=38.5GiB | 28.4% done | ETA(epoch): 0s + Train loss: 0.657668 (2651.6s) ETA: 17607min + Val loss: 0.707203 [t_0.0-0.2=0.9237 t_0.2-0.4=0.7472 t_0.4-0.6=0.6668 t_0.6-0.8=0.5612 t_0.8-1.0=0.5620] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 23.1/188.5 GiB (12.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 142/499 (28% done) --- + [Epoch 142][10/200] loss=0.601542 avg=0.642059 VRAM=38.6GiB | 28.4% done | ETA(epoch): 2518s + [Epoch 142][20/200] loss=0.595697 avg=0.628501 VRAM=38.5GiB | 28.4% done | ETA(epoch): 2385s + [Epoch 142][30/200] loss=0.532061 avg=0.639906 VRAM=38.6GiB | 28.4% done | ETA(epoch): 2253s + [Epoch 142][40/200] loss=0.599219 avg=0.636659 VRAM=38.5GiB | 28.4% done | ETA(epoch): 2120s + [Epoch 142][50/200] loss=0.587251 avg=0.636091 VRAM=38.6GiB | 28.4% done | ETA(epoch): 1989s + [Epoch 142][60/200] loss=0.680499 avg=0.636252 VRAM=38.5GiB | 28.5% done | ETA(epoch): 1856s + [Epoch 142][70/200] loss=0.569856 avg=0.643451 VRAM=38.6GiB | 28.5% done | ETA(epoch): 1724s + [Epoch 142][80/200] loss=0.718756 avg=0.644232 VRAM=38.5GiB | 28.5% done | ETA(epoch): 1595s + [Epoch 142][90/200] loss=0.616899 avg=0.642092 VRAM=38.6GiB | 28.5% done | ETA(epoch): 1461s + [Epoch 142][100/200] loss=0.696711 avg=0.647150 VRAM=38.5GiB | 28.5% done | ETA(epoch): 1328s + [Epoch 142][110/200] loss=0.620775 avg=0.647786 VRAM=38.6GiB | 28.5% done | ETA(epoch): 1195s + [Epoch 142][120/200] loss=0.632850 avg=0.650332 VRAM=38.5GiB | 28.5% done | ETA(epoch): 1062s + [Epoch 142][130/200] loss=0.591838 avg=0.651584 VRAM=38.6GiB | 28.5% done | ETA(epoch): 929s + [Epoch 142][140/200] loss=0.634112 avg=0.647578 VRAM=38.5GiB | 28.5% done | ETA(epoch): 797s + [Epoch 142][150/200] loss=0.720168 avg=0.646868 VRAM=38.6GiB | 28.5% done | ETA(epoch): 664s + [Epoch 142][160/200] loss=0.609324 avg=0.647817 VRAM=38.5GiB | 28.6% done | ETA(epoch): 531s + [Epoch 142][170/200] loss=0.685182 avg=0.649548 VRAM=38.6GiB | 28.6% done | ETA(epoch): 399s + [Epoch 142][180/200] loss=0.542546 avg=0.649190 VRAM=38.5GiB | 28.6% done | ETA(epoch): 266s + [Epoch 142][190/200] loss=0.685860 avg=0.648990 VRAM=38.6GiB | 28.6% done | ETA(epoch): 133s + [Epoch 142][200/200] loss=0.667438 avg=0.651185 VRAM=38.5GiB | 28.6% done | ETA(epoch): 0s + Train loss: 0.651185 (2657.3s) ETA: 17556min + Val loss: 0.695814 [t_0.0-0.2=0.9395 t_0.2-0.4=0.7351 t_0.4-0.6=0.6616 t_0.6-0.8=0.5475 t_0.8-1.0=0.5732] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 143/499 (29% done) --- + [Epoch 143][10/200] loss=0.618423 avg=0.648323 VRAM=38.6GiB | 28.6% done | ETA(epoch): 2517s + [Epoch 143][20/200] loss=0.656169 avg=0.658388 VRAM=38.5GiB | 28.6% done | ETA(epoch): 2388s + [Epoch 143][30/200] loss=0.658914 avg=0.653358 VRAM=38.6GiB | 28.6% done | ETA(epoch): 2254s + [Epoch 143][40/200] loss=0.575935 avg=0.658683 VRAM=38.5GiB | 28.6% done | ETA(epoch): 2132s + [Epoch 143][50/200] loss=0.725629 avg=0.660511 VRAM=38.6GiB | 28.6% done | ETA(epoch): 1997s + [Epoch 143][60/200] loss=0.698593 avg=0.664502 VRAM=38.5GiB | 28.7% done | ETA(epoch): 1862s + [Epoch 143][70/200] loss=0.598856 avg=0.660868 VRAM=38.6GiB | 28.7% done | ETA(epoch): 1728s + [Epoch 143][80/200] loss=0.849016 avg=0.658778 VRAM=38.5GiB | 28.7% done | ETA(epoch): 1595s + [Epoch 143][90/200] loss=0.588728 avg=0.659777 VRAM=38.6GiB | 28.7% done | ETA(epoch): 1461s + [Epoch 143][100/200] loss=0.660889 avg=0.661884 VRAM=38.5GiB | 28.7% done | ETA(epoch): 1328s + [Epoch 143][110/200] loss=0.693653 avg=0.660344 VRAM=38.6GiB | 28.7% done | ETA(epoch): 1195s + [Epoch 143][120/200] loss=0.709337 avg=0.660501 VRAM=38.5GiB | 28.7% done | ETA(epoch): 1062s + [Epoch 143][130/200] loss=0.591535 avg=0.659128 VRAM=38.6GiB | 28.7% done | ETA(epoch): 932s + [Epoch 143][140/200] loss=0.640872 avg=0.659893 VRAM=38.5GiB | 28.7% done | ETA(epoch): 798s + [Epoch 143][150/200] loss=0.716564 avg=0.662158 VRAM=38.6GiB | 28.7% done | ETA(epoch): 665s + [Epoch 143][160/200] loss=0.752828 avg=0.662303 VRAM=38.5GiB | 28.8% done | ETA(epoch): 532s + [Epoch 143][170/200] loss=0.662515 avg=0.661821 VRAM=38.6GiB | 28.8% done | ETA(epoch): 399s + [Epoch 143][180/200] loss=0.629141 avg=0.661875 VRAM=38.5GiB | 28.8% done | ETA(epoch): 266s + [Epoch 143][190/200] loss=0.567875 avg=0.661843 VRAM=38.6GiB | 28.8% done | ETA(epoch): 133s + [Epoch 143][200/200] loss=0.607155 avg=0.661699 VRAM=38.5GiB | 28.8% done | ETA(epoch): 0s + Train loss: 0.661699 (2658.5s) ETA: 17505min + Val loss: 0.705969 [t_0.0-0.2=0.9422 t_0.2-0.4=0.7545 t_0.4-0.6=0.6339 t_0.6-0.8=0.5782 t_0.8-1.0=0.5776] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 144/499 (29% done) --- + [Epoch 144][10/200] loss=0.563492 avg=0.628295 VRAM=38.6GiB | 28.8% done | ETA(epoch): 2515s + [Epoch 144][20/200] loss=0.667061 avg=0.639213 VRAM=38.5GiB | 28.8% done | ETA(epoch): 2385s + [Epoch 144][30/200] loss=0.609730 avg=0.633375 VRAM=38.6GiB | 28.8% done | ETA(epoch): 2253s + [Epoch 144][40/200] loss=0.663641 avg=0.631874 VRAM=38.5GiB | 28.8% done | ETA(epoch): 2121s + [Epoch 144][50/200] loss=0.595995 avg=0.632348 VRAM=38.6GiB | 28.8% done | ETA(epoch): 1988s + [Epoch 144][60/200] loss=0.665175 avg=0.638474 VRAM=38.5GiB | 28.9% done | ETA(epoch): 1856s + [Epoch 144][70/200] loss=0.605538 avg=0.645906 VRAM=38.6GiB | 28.9% done | ETA(epoch): 1723s + [Epoch 144][80/200] loss=0.720560 avg=0.650343 VRAM=38.5GiB | 28.9% done | ETA(epoch): 1591s + [Epoch 144][90/200] loss=0.642984 avg=0.647323 VRAM=38.6GiB | 28.9% done | ETA(epoch): 1466s + [Epoch 144][100/200] loss=0.724358 avg=0.649764 VRAM=38.5GiB | 28.9% done | ETA(epoch): 1332s + [Epoch 144][110/200] loss=0.665478 avg=0.650010 VRAM=38.6GiB | 28.9% done | ETA(epoch): 1198s + [Epoch 144][120/200] loss=0.676183 avg=0.647162 VRAM=38.5GiB | 28.9% done | ETA(epoch): 1065s + [Epoch 144][130/200] loss=0.640356 avg=0.647446 VRAM=38.6GiB | 28.9% done | ETA(epoch): 931s + [Epoch 144][140/200] loss=0.644185 avg=0.650074 VRAM=38.5GiB | 28.9% done | ETA(epoch): 798s + [Epoch 144][150/200] loss=0.752416 avg=0.650079 VRAM=38.6GiB | 28.9% done | ETA(epoch): 665s + [Epoch 144][160/200] loss=0.665517 avg=0.648326 VRAM=38.5GiB | 29.0% done | ETA(epoch): 532s + [Epoch 144][170/200] loss=0.579496 avg=0.648525 VRAM=38.6GiB | 29.0% done | ETA(epoch): 399s + [Epoch 144][180/200] loss=0.640168 avg=0.648730 VRAM=38.5GiB | 29.0% done | ETA(epoch): 266s + [Epoch 144][190/200] loss=0.651085 avg=0.648866 VRAM=38.6GiB | 29.0% done | ETA(epoch): 133s + [Epoch 144][200/200] loss=0.588088 avg=0.648106 VRAM=38.5GiB | 29.0% done | ETA(epoch): 0s + Train loss: 0.648106 (2657.7s) ETA: 17454min + Val loss: 0.695575 [t_0.0-0.2=0.9417 t_0.2-0.4=0.7578 t_0.4-0.6=0.6257 t_0.6-0.8=0.5573 t_0.8-1.0=0.5898] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 145/499 (29% done) --- + [Epoch 145][10/200] loss=0.595616 avg=0.627646 VRAM=38.6GiB | 29.0% done | ETA(epoch): 2513s + [Epoch 145][20/200] loss=0.641018 avg=0.657715 VRAM=38.5GiB | 29.0% done | ETA(epoch): 2383s + [Epoch 145][30/200] loss=0.625763 avg=0.658275 VRAM=38.6GiB | 29.0% done | ETA(epoch): 2253s + [Epoch 145][40/200] loss=0.772640 avg=0.649764 VRAM=38.5GiB | 29.0% done | ETA(epoch): 2120s + [Epoch 145][50/200] loss=0.614914 avg=0.638622 VRAM=38.6GiB | 29.0% done | ETA(epoch): 1988s + [Epoch 145][60/200] loss=0.758929 avg=0.642392 VRAM=38.5GiB | 29.1% done | ETA(epoch): 1855s + [Epoch 145][70/200] loss=0.628690 avg=0.645924 VRAM=38.6GiB | 29.1% done | ETA(epoch): 1723s + [Epoch 145][80/200] loss=0.646716 avg=0.647519 VRAM=38.5GiB | 29.1% done | ETA(epoch): 1590s + [Epoch 145][90/200] loss=0.637527 avg=0.648107 VRAM=38.6GiB | 29.1% done | ETA(epoch): 1458s + [Epoch 145][100/200] loss=0.864022 avg=0.653264 VRAM=38.5GiB | 29.1% done | ETA(epoch): 1325s + [Epoch 145][110/200] loss=0.747457 avg=0.657351 VRAM=38.6GiB | 29.1% done | ETA(epoch): 1193s + [Epoch 145][120/200] loss=0.558870 avg=0.656162 VRAM=38.5GiB | 29.1% done | ETA(epoch): 1060s + [Epoch 145][130/200] loss=0.691628 avg=0.657994 VRAM=38.6GiB | 29.1% done | ETA(epoch): 928s + [Epoch 145][140/200] loss=0.534892 avg=0.654088 VRAM=38.5GiB | 29.1% done | ETA(epoch): 795s + [Epoch 145][150/200] loss=0.635583 avg=0.654307 VRAM=38.6GiB | 29.1% done | ETA(epoch): 664s + [Epoch 145][160/200] loss=0.613140 avg=0.653235 VRAM=38.5GiB | 29.2% done | ETA(epoch): 531s + [Epoch 145][170/200] loss=0.657628 avg=0.653300 VRAM=38.6GiB | 29.2% done | ETA(epoch): 398s + [Epoch 145][180/200] loss=0.630673 avg=0.653882 VRAM=38.5GiB | 29.2% done | ETA(epoch): 265s + [Epoch 145][190/200] loss=0.692350 avg=0.655551 VRAM=38.6GiB | 29.2% done | ETA(epoch): 133s + [Epoch 145][200/200] loss=0.808318 avg=0.655551 VRAM=38.5GiB | 29.2% done | ETA(epoch): 0s + Train loss: 0.655551 (2653.9s) ETA: 17403min + Val loss: 0.664784 [t_0.0-0.2=0.9101 t_0.2-0.4=0.7585 t_0.4-0.6=0.6480 t_0.6-0.8=0.5598 t_0.8-1.0=0.5598] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 146/499 (29% done) --- + [Epoch 146][10/200] loss=0.611190 avg=0.680884 VRAM=38.6GiB | 29.2% done | ETA(epoch): 2519s + [Epoch 146][20/200] loss=0.622505 avg=0.648405 VRAM=38.5GiB | 29.2% done | ETA(epoch): 2386s + [Epoch 146][30/200] loss=0.707704 avg=0.660649 VRAM=38.6GiB | 29.2% done | ETA(epoch): 2254s + [Epoch 146][40/200] loss=0.588745 avg=0.653063 VRAM=38.5GiB | 29.2% done | ETA(epoch): 2121s + [Epoch 146][50/200] loss=0.604277 avg=0.657585 VRAM=38.6GiB | 29.2% done | ETA(epoch): 1989s + [Epoch 146][60/200] loss=0.512751 avg=0.654963 VRAM=38.5GiB | 29.3% done | ETA(epoch): 1856s + [Epoch 146][70/200] loss=0.678954 avg=0.652066 VRAM=38.6GiB | 29.3% done | ETA(epoch): 1724s + [Epoch 146][80/200] loss=0.641511 avg=0.649731 VRAM=38.5GiB | 29.3% done | ETA(epoch): 1591s + [Epoch 146][90/200] loss=0.847895 avg=0.654559 VRAM=38.6GiB | 29.3% done | ETA(epoch): 1459s + [Epoch 146][100/200] loss=0.702409 avg=0.659979 VRAM=38.5GiB | 29.3% done | ETA(epoch): 1326s + [Epoch 146][110/200] loss=0.614945 avg=0.663024 VRAM=38.6GiB | 29.3% done | ETA(epoch): 1195s + [Epoch 146][120/200] loss=0.752743 avg=0.665650 VRAM=38.5GiB | 29.3% done | ETA(epoch): 1062s + [Epoch 146][130/200] loss=0.584135 avg=0.667567 VRAM=38.6GiB | 29.3% done | ETA(epoch): 929s + [Epoch 146][140/200] loss=0.728319 avg=0.666176 VRAM=38.5GiB | 29.3% done | ETA(epoch): 796s + [Epoch 146][150/200] loss=0.677218 avg=0.664755 VRAM=38.6GiB | 29.3% done | ETA(epoch): 664s + [Epoch 146][160/200] loss=0.551560 avg=0.662953 VRAM=38.5GiB | 29.4% done | ETA(epoch): 531s + [Epoch 146][170/200] loss=0.588493 avg=0.661068 VRAM=38.6GiB | 29.4% done | ETA(epoch): 398s + [Epoch 146][180/200] loss=0.651684 avg=0.661269 VRAM=38.5GiB | 29.4% done | ETA(epoch): 265s + [Epoch 146][190/200] loss=0.860429 avg=0.663962 VRAM=38.6GiB | 29.4% done | ETA(epoch): 133s + [Epoch 146][200/200] loss=0.642141 avg=0.663577 VRAM=38.5GiB | 29.4% done | ETA(epoch): 0s + Train loss: 0.663577 (2658.0s) ETA: 17353min + Val loss: 0.663131 [t_0.0-0.2=0.9521 t_0.2-0.4=0.7280 t_0.4-0.6=0.6436 t_0.6-0.8=0.5712 t_0.8-1.0=0.5642] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 147/499 (29% done) --- + [Epoch 147][10/200] loss=0.828186 avg=0.679155 VRAM=38.6GiB | 29.4% done | ETA(epoch): 2514s + [Epoch 147][20/200] loss=0.677881 avg=0.656761 VRAM=38.5GiB | 29.4% done | ETA(epoch): 2386s + [Epoch 147][30/200] loss=0.646242 avg=0.665577 VRAM=38.6GiB | 29.4% done | ETA(epoch): 2252s + [Epoch 147][40/200] loss=0.794473 avg=0.679335 VRAM=38.5GiB | 29.4% done | ETA(epoch): 2120s + [Epoch 147][50/200] loss=0.708865 avg=0.684450 VRAM=38.6GiB | 29.4% done | ETA(epoch): 1987s + [Epoch 147][60/200] loss=0.669676 avg=0.681805 VRAM=38.5GiB | 29.5% done | ETA(epoch): 1854s + [Epoch 147][70/200] loss=0.690738 avg=0.683596 VRAM=38.6GiB | 29.5% done | ETA(epoch): 1724s + [Epoch 147][80/200] loss=0.715304 avg=0.681122 VRAM=38.5GiB | 29.5% done | ETA(epoch): 1591s + [Epoch 147][90/200] loss=0.713518 avg=0.677057 VRAM=38.6GiB | 29.5% done | ETA(epoch): 1458s + [Epoch 147][100/200] loss=0.705813 avg=0.674828 VRAM=38.5GiB | 29.5% done | ETA(epoch): 1325s + [Epoch 147][110/200] loss=0.645333 avg=0.675472 VRAM=38.6GiB | 29.5% done | ETA(epoch): 1193s + [Epoch 147][120/200] loss=0.725116 avg=0.674457 VRAM=38.5GiB | 29.5% done | ETA(epoch): 1060s + [Epoch 147][130/200] loss=0.719798 avg=0.669528 VRAM=38.6GiB | 29.5% done | ETA(epoch): 928s + [Epoch 147][140/200] loss=0.675527 avg=0.672886 VRAM=38.5GiB | 29.5% done | ETA(epoch): 795s + [Epoch 147][150/200] loss=0.638008 avg=0.671227 VRAM=38.6GiB | 29.5% done | ETA(epoch): 663s + [Epoch 147][160/200] loss=0.723087 avg=0.671751 VRAM=38.5GiB | 29.6% done | ETA(epoch): 530s + [Epoch 147][170/200] loss=0.609868 avg=0.669930 VRAM=38.6GiB | 29.6% done | ETA(epoch): 397s + [Epoch 147][180/200] loss=0.657547 avg=0.669281 VRAM=38.5GiB | 29.6% done | ETA(epoch): 265s + [Epoch 147][190/200] loss=0.519897 avg=0.667856 VRAM=38.6GiB | 29.6% done | ETA(epoch): 132s + [Epoch 147][200/200] loss=0.595346 avg=0.665910 VRAM=38.5GiB | 29.6% done | ETA(epoch): 0s + Train loss: 0.665910 (2649.8s) ETA: 17301min + Val loss: 0.682789 [t_0.0-0.2=0.9216 t_0.2-0.4=0.7568 t_0.4-0.6=0.6334 t_0.6-0.8=0.5757 t_0.8-1.0=0.5704] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 148/499 (30% done) --- + [Epoch 148][10/200] loss=0.710679 avg=0.672731 VRAM=38.6GiB | 29.6% done | ETA(epoch): 2522s + [Epoch 148][20/200] loss=0.691278 avg=0.671240 VRAM=38.5GiB | 29.6% done | ETA(epoch): 2386s + [Epoch 148][30/200] loss=0.650265 avg=0.654408 VRAM=38.6GiB | 29.6% done | ETA(epoch): 2253s + [Epoch 148][40/200] loss=0.579064 avg=0.656428 VRAM=38.5GiB | 29.6% done | ETA(epoch): 2121s + [Epoch 148][50/200] loss=0.721311 avg=0.660688 VRAM=38.6GiB | 29.6% done | ETA(epoch): 1988s + [Epoch 148][60/200] loss=0.577077 avg=0.662330 VRAM=38.5GiB | 29.7% done | ETA(epoch): 1856s + [Epoch 148][70/200] loss=0.658106 avg=0.659341 VRAM=38.6GiB | 29.7% done | ETA(epoch): 1723s + [Epoch 148][80/200] loss=0.882124 avg=0.664157 VRAM=38.5GiB | 29.7% done | ETA(epoch): 1590s + [Epoch 148][90/200] loss=0.577508 avg=0.661913 VRAM=38.6GiB | 29.7% done | ETA(epoch): 1458s + [Epoch 148][100/200] loss=0.698603 avg=0.664404 VRAM=38.5GiB | 29.7% done | ETA(epoch): 1325s + [Epoch 148][110/200] loss=0.638113 avg=0.659903 VRAM=38.6GiB | 29.7% done | ETA(epoch): 1193s + [Epoch 148][120/200] loss=0.756294 avg=0.663761 VRAM=38.5GiB | 29.7% done | ETA(epoch): 1060s + [Epoch 148][130/200] loss=0.591285 avg=0.659350 VRAM=38.6GiB | 29.7% done | ETA(epoch): 928s + [Epoch 148][140/200] loss=0.659332 avg=0.660304 VRAM=38.5GiB | 29.7% done | ETA(epoch): 795s + [Epoch 148][150/200] loss=0.684019 avg=0.662364 VRAM=38.6GiB | 29.8% done | ETA(epoch): 662s + [Epoch 148][160/200] loss=0.714965 avg=0.663109 VRAM=38.5GiB | 29.8% done | ETA(epoch): 530s + [Epoch 148][170/200] loss=0.602086 avg=0.660650 VRAM=38.6GiB | 29.8% done | ETA(epoch): 397s + [Epoch 148][180/200] loss=0.657485 avg=0.660447 VRAM=38.5GiB | 29.8% done | ETA(epoch): 265s + [Epoch 148][190/200] loss=0.552417 avg=0.658679 VRAM=38.6GiB | 29.8% done | ETA(epoch): 132s + [Epoch 148][200/200] loss=0.748714 avg=0.656516 VRAM=38.5GiB | 29.8% done | ETA(epoch): 0s + Train loss: 0.656516 (2649.9s) ETA: 17250min + Val loss: 0.691036 [t_0.0-0.2=0.9273 t_0.2-0.4=0.7491 t_0.4-0.6=0.6445 t_0.6-0.8=0.5598 t_0.8-1.0=0.5742] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 149/499 (30% done) --- + [Epoch 149][10/200] loss=0.710763 avg=0.696605 VRAM=38.6GiB | 29.8% done | ETA(epoch): 2518s + [Epoch 149][20/200] loss=0.753887 avg=0.706529 VRAM=38.5GiB | 29.8% done | ETA(epoch): 2385s + [Epoch 149][30/200] loss=0.582768 avg=0.689269 VRAM=38.6GiB | 29.8% done | ETA(epoch): 2253s + [Epoch 149][40/200] loss=0.695537 avg=0.679591 VRAM=38.5GiB | 29.8% done | ETA(epoch): 2120s + [Epoch 149][50/200] loss=0.655402 avg=0.665936 VRAM=38.6GiB | 29.8% done | ETA(epoch): 1987s + [Epoch 149][60/200] loss=0.679993 avg=0.663327 VRAM=38.5GiB | 29.9% done | ETA(epoch): 1854s + [Epoch 149][70/200] loss=0.720892 avg=0.658195 VRAM=38.6GiB | 29.9% done | ETA(epoch): 1722s + [Epoch 149][80/200] loss=0.612695 avg=0.654549 VRAM=38.5GiB | 29.9% done | ETA(epoch): 1589s + [Epoch 149][90/200] loss=0.696220 avg=0.655386 VRAM=38.6GiB | 29.9% done | ETA(epoch): 1460s + [Epoch 149][100/200] loss=0.617530 avg=0.659343 VRAM=38.5GiB | 29.9% done | ETA(epoch): 1327s + [Epoch 149][110/200] loss=0.569430 avg=0.661280 VRAM=38.6GiB | 29.9% done | ETA(epoch): 1194s + [Epoch 149][120/200] loss=0.786828 avg=0.664502 VRAM=38.5GiB | 29.9% done | ETA(epoch): 1061s + [Epoch 149][130/200] loss=0.793147 avg=0.665447 VRAM=38.6GiB | 29.9% done | ETA(epoch): 929s + [Epoch 149][140/200] loss=0.599805 avg=0.667215 VRAM=38.5GiB | 29.9% done | ETA(epoch): 796s + [Epoch 149][150/200] loss=0.647260 avg=0.667316 VRAM=38.6GiB | 29.9% done | ETA(epoch): 663s + [Epoch 149][160/200] loss=0.616210 avg=0.665035 VRAM=38.5GiB | 30.0% done | ETA(epoch): 530s + [Epoch 149][170/200] loss=0.649134 avg=0.663294 VRAM=38.6GiB | 30.0% done | ETA(epoch): 398s + [Epoch 149][180/200] loss=0.804877 avg=0.663133 VRAM=38.5GiB | 30.0% done | ETA(epoch): 265s + [Epoch 149][190/200] loss=0.823007 avg=0.662508 VRAM=38.6GiB | 30.0% done | ETA(epoch): 133s + [Epoch 149][200/200] loss=0.620329 avg=0.661118 VRAM=38.5GiB | 30.0% done | ETA(epoch): 0s + Train loss: 0.661118 (2654.5s) ETA: 17199min + Val loss: 0.684967 [t_0.0-0.2=0.9411 t_0.2-0.4=0.7437 t_0.4-0.6=0.6411 t_0.6-0.8=0.5639 t_0.8-1.0=0.5615] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 150/499 (30% done) --- + [MilestoneVis] train_0 step 30000 ✓ + [MilestoneGrid] train_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 30000 ✓ + [MilestoneGrid] train_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 30000 ✓ + [MilestoneGrid] val_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 30000 ✓ + [MilestoneGrid] val_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 30000 ✓ + [MilestoneGrid] test_0 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 30000 ✓ + [MilestoneGrid] test_1 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 30000 ✓ + [MilestoneGrid] test_2 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 30000 ✓ + [MilestoneGrid] test_3 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 30000 ✓ + [MilestoneGrid] test_4 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 30000 ✓ + [MilestoneGrid] test_5 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_5.png + [MilestoneVis] test_6 step 30000 ✓ + [MilestoneGrid] test_6 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_6.png + [MilestoneVis] test_7 step 30000 ✓ + [MilestoneGrid] test_7 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_7.png + [MilestoneVis] test_8 step 30000 ✓ + [MilestoneGrid] test_8 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_8.png + [MilestoneVis] test_9 step 30000 ✓ + [MilestoneGrid] test_9 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_9.png + [MilestoneVis] test_10 step 30000 ✓ + [MilestoneGrid] test_10 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_10.png + [MilestoneVis] test_11 step 30000 ✓ + [MilestoneGrid] test_11 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_11.png + [MilestoneVis] test_12 step 30000 ✓ + [MilestoneGrid] test_12 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_12.png + [MilestoneVis] test_13 step 30000 ✓ + [MilestoneGrid] test_13 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_13.png + [MilestoneVis] test_14 step 30000 ✓ + [MilestoneGrid] test_14 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_14.png + [MilestoneVis] test_15 step 30000 ✓ + [MilestoneGrid] test_15 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_15.png + [MilestoneVis] test_16 step 30000 ✓ + [MilestoneGrid] test_16 → output/lora_rank_128_mlp_1k_h100/milestone_vis/milestone_grid_test_16.png + [Epoch 150][10/200] loss=0.717503 avg=0.691585 VRAM=38.6GiB | 30.0% done | ETA(epoch): 43947s + [Epoch 150][20/200] loss=0.545448 avg=0.658269 VRAM=38.5GiB | 30.0% done | ETA(epoch): 22010s + [Epoch 150][30/200] loss=0.601289 avg=0.664974 VRAM=38.6GiB | 30.0% done | ETA(epoch): 14608s + [Epoch 150][40/200] loss=0.560673 avg=0.664861 VRAM=38.5GiB | 30.0% done | ETA(epoch): 10841s + [Epoch 150][50/200] loss=0.594060 avg=0.670608 VRAM=38.6GiB | 30.0% done | ETA(epoch): 8528s + [Epoch 150][60/200] loss=0.665568 avg=0.665156 VRAM=38.5GiB | 30.1% done | ETA(epoch): 6942s + [Epoch 150][70/200] loss=0.577101 avg=0.663900 VRAM=38.6GiB | 30.1% done | ETA(epoch): 5772s + [Epoch 150][80/200] loss=0.667870 avg=0.667925 VRAM=38.5GiB | 30.1% done | ETA(epoch): 4861s + [Epoch 150][90/200] loss=0.648180 avg=0.670295 VRAM=38.6GiB | 30.1% done | ETA(epoch): 4123s + [Epoch 150][100/200] loss=0.649927 avg=0.670383 VRAM=38.5GiB | 30.1% done | ETA(epoch): 3506s + [Epoch 150][110/200] loss=0.615249 avg=0.667089 VRAM=38.6GiB | 30.1% done | ETA(epoch): 2976s + [Epoch 150][120/200] loss=0.726427 avg=0.666058 VRAM=38.5GiB | 30.1% done | ETA(epoch): 2514s + [Epoch 150][130/200] loss=0.680559 avg=0.666044 VRAM=38.6GiB | 30.1% done | ETA(epoch): 2102s + [Epoch 150][140/200] loss=0.728928 avg=0.664091 VRAM=38.5GiB | 30.1% done | ETA(epoch): 1730s + [Epoch 150][150/200] loss=0.623970 avg=0.661245 VRAM=38.6GiB | 30.1% done | ETA(epoch): 1389s + [Epoch 150][160/200] loss=0.668675 avg=0.661333 VRAM=38.5GiB | 30.2% done | ETA(epoch): 1076s + [Epoch 150][170/200] loss=0.822185 avg=0.661173 VRAM=38.6GiB | 30.2% done | ETA(epoch): 783s + [Epoch 150][180/200] loss=0.643960 avg=0.661932 VRAM=38.5GiB | 30.2% done | ETA(epoch): 508s + [Epoch 150][190/200] loss=0.885942 avg=0.663359 VRAM=38.6GiB | 30.2% done | ETA(epoch): 247s + [Epoch 150][200/200] loss=0.707916 avg=0.661029 VRAM=38.5GiB | 30.2% done | ETA(epoch): 0s + Train loss: 0.661029 (4833.4s) ETA: 17233min + Val loss: 0.700271 [t_0.0-0.2=0.9194 t_0.2-0.4=0.7482 t_0.4-0.6=0.6339 t_0.6-0.8=0.5677 t_0.8-1.0=0.5705] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 151/499 (30% done) --- + [Epoch 151][10/200] loss=0.556804 avg=0.680873 VRAM=38.6GiB | 30.2% done | ETA(epoch): 2513s + [Epoch 151][20/200] loss=0.581202 avg=0.648891 VRAM=38.5GiB | 30.2% done | ETA(epoch): 2382s + [Epoch 151][30/200] loss=0.646002 avg=0.655000 VRAM=38.6GiB | 30.2% done | ETA(epoch): 2250s + [Epoch 151][40/200] loss=0.664149 avg=0.658720 VRAM=38.5GiB | 30.2% done | ETA(epoch): 2118s + [Epoch 151][50/200] loss=0.705091 avg=0.664429 VRAM=38.6GiB | 30.2% done | ETA(epoch): 1985s + [Epoch 151][60/200] loss=0.770777 avg=0.660953 VRAM=38.5GiB | 30.3% done | ETA(epoch): 1853s + [Epoch 151][70/200] loss=0.692264 avg=0.661810 VRAM=38.6GiB | 30.3% done | ETA(epoch): 1721s + [Epoch 151][80/200] loss=0.587507 avg=0.661757 VRAM=38.5GiB | 30.3% done | ETA(epoch): 1588s + [Epoch 151][90/200] loss=0.656771 avg=0.659514 VRAM=38.6GiB | 30.3% done | ETA(epoch): 1456s + [Epoch 151][100/200] loss=0.684625 avg=0.661814 VRAM=38.5GiB | 30.3% done | ETA(epoch): 1324s + [Epoch 151][110/200] loss=0.759077 avg=0.665151 VRAM=38.6GiB | 30.3% done | ETA(epoch): 1191s + [Epoch 151][120/200] loss=0.685253 avg=0.667212 VRAM=38.5GiB | 30.3% done | ETA(epoch): 1061s + [Epoch 151][130/200] loss=0.761918 avg=0.668252 VRAM=38.6GiB | 30.3% done | ETA(epoch): 928s + [Epoch 151][140/200] loss=0.590823 avg=0.665645 VRAM=38.5GiB | 30.3% done | ETA(epoch): 796s + [Epoch 151][150/200] loss=0.587627 avg=0.664571 VRAM=38.6GiB | 30.3% done | ETA(epoch): 663s + [Epoch 151][160/200] loss=0.648726 avg=0.662586 VRAM=38.5GiB | 30.4% done | ETA(epoch): 530s + [Epoch 151][170/200] loss=0.787598 avg=0.664163 VRAM=38.6GiB | 30.4% done | ETA(epoch): 398s + [Epoch 151][180/200] loss=0.550954 avg=0.665172 VRAM=38.5GiB | 30.4% done | ETA(epoch): 265s + [Epoch 151][190/200] loss=0.659605 avg=0.663566 VRAM=38.6GiB | 30.4% done | ETA(epoch): 133s + [Epoch 151][200/200] loss=0.710649 avg=0.663711 VRAM=38.5GiB | 30.4% done | ETA(epoch): 0s + Train loss: 0.663711 (2651.1s) ETA: 17181min + Val loss: 0.690864 [t_0.0-0.2=0.9273 t_0.2-0.4=0.7566 t_0.4-0.6=0.6440 t_0.6-0.8=0.5395 t_0.8-1.0=0.5725] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 24.8/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 152/499 (30% done) --- + [Epoch 152][10/200] loss=0.694960 avg=0.675620 VRAM=38.6GiB | 30.4% done | ETA(epoch): 2520s + [Epoch 152][20/200] loss=0.594069 avg=0.668245 VRAM=38.5GiB | 30.4% done | ETA(epoch): 2386s + [Epoch 152][30/200] loss=0.620005 avg=0.649375 VRAM=38.6GiB | 30.4% done | ETA(epoch): 2251s + [Epoch 152][40/200] loss=0.677251 avg=0.665144 VRAM=38.5GiB | 30.4% done | ETA(epoch): 2119s + [Epoch 152][50/200] loss=0.562356 avg=0.672969 VRAM=38.6GiB | 30.4% done | ETA(epoch): 1986s + [Epoch 152][60/200] loss=0.769989 avg=0.672687 VRAM=38.5GiB | 30.5% done | ETA(epoch): 1854s + [Epoch 152][70/200] loss=0.696352 avg=0.674649 VRAM=38.6GiB | 30.5% done | ETA(epoch): 1722s + [Epoch 152][80/200] loss=0.596945 avg=0.670928 VRAM=38.5GiB | 30.5% done | ETA(epoch): 1589s + [Epoch 152][90/200] loss=0.581336 avg=0.664983 VRAM=38.6GiB | 30.5% done | ETA(epoch): 1463s + [Epoch 152][100/200] loss=0.750380 avg=0.664082 VRAM=38.5GiB | 30.5% done | ETA(epoch): 1329s + [Epoch 152][110/200] loss=0.585432 avg=0.664687 VRAM=38.6GiB | 30.5% done | ETA(epoch): 1196s + [Epoch 152][120/200] loss=0.550464 avg=0.665086 VRAM=38.5GiB | 30.5% done | ETA(epoch): 1063s + [Epoch 152][130/200] loss=0.664405 avg=0.664759 VRAM=38.6GiB | 30.5% done | ETA(epoch): 930s + [Epoch 152][140/200] loss=0.674597 avg=0.662573 VRAM=38.5GiB | 30.5% done | ETA(epoch): 797s + [Epoch 152][150/200] loss=0.545196 avg=0.663099 VRAM=38.6GiB | 30.6% done | ETA(epoch): 664s + [Epoch 152][160/200] loss=0.640355 avg=0.662181 VRAM=38.5GiB | 30.6% done | ETA(epoch): 531s + [Epoch 152][170/200] loss=0.672079 avg=0.661223 VRAM=38.6GiB | 30.6% done | ETA(epoch): 398s + [Epoch 152][180/200] loss=0.732948 avg=0.662562 VRAM=38.5GiB | 30.6% done | ETA(epoch): 266s + [Epoch 152][190/200] loss=0.647798 avg=0.663283 VRAM=38.6GiB | 30.6% done | ETA(epoch): 133s + [Epoch 152][200/200] loss=0.587693 avg=0.661786 VRAM=38.5GiB | 30.6% done | ETA(epoch): 0s + Train loss: 0.661786 (2655.3s) ETA: 17129min + Val loss: 0.676630 [t_0.0-0.2=0.9228 t_0.2-0.4=0.7446 t_0.4-0.6=0.6500 t_0.6-0.8=0.5664 t_0.8-1.0=0.5757] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 153/499 (31% done) --- + [Epoch 153][10/200] loss=0.671023 avg=0.674527 VRAM=38.6GiB | 30.6% done | ETA(epoch): 2517s + [Epoch 153][20/200] loss=0.480490 avg=0.651880 VRAM=38.5GiB | 30.6% done | ETA(epoch): 2383s + [Epoch 153][30/200] loss=0.630420 avg=0.655248 VRAM=38.6GiB | 30.6% done | ETA(epoch): 2251s + [Epoch 153][40/200] loss=0.627588 avg=0.666054 VRAM=38.5GiB | 30.6% done | ETA(epoch): 2118s + [Epoch 153][50/200] loss=0.662281 avg=0.659888 VRAM=38.6GiB | 30.6% done | ETA(epoch): 1986s + [Epoch 153][60/200] loss=0.669593 avg=0.655021 VRAM=38.5GiB | 30.7% done | ETA(epoch): 1853s + [Epoch 153][70/200] loss=0.567032 avg=0.652566 VRAM=38.6GiB | 30.7% done | ETA(epoch): 1721s + [Epoch 153][80/200] loss=0.629749 avg=0.653656 VRAM=38.5GiB | 30.7% done | ETA(epoch): 1589s + [Epoch 153][90/200] loss=0.710277 avg=0.659143 VRAM=38.6GiB | 30.7% done | ETA(epoch): 1456s + [Epoch 153][100/200] loss=0.629417 avg=0.657552 VRAM=38.5GiB | 30.7% done | ETA(epoch): 1324s + [Epoch 153][110/200] loss=0.694882 avg=0.660418 VRAM=38.6GiB | 30.7% done | ETA(epoch): 1192s + [Epoch 153][120/200] loss=0.646775 avg=0.659679 VRAM=38.5GiB | 30.7% done | ETA(epoch): 1059s + [Epoch 153][130/200] loss=0.581766 avg=0.658602 VRAM=38.6GiB | 30.7% done | ETA(epoch): 927s + [Epoch 153][140/200] loss=0.588553 avg=0.657671 VRAM=38.5GiB | 30.7% done | ETA(epoch): 795s + [Epoch 153][150/200] loss=0.622574 avg=0.657527 VRAM=38.6GiB | 30.8% done | ETA(epoch): 663s + [Epoch 153][160/200] loss=0.589151 avg=0.655546 VRAM=38.5GiB | 30.8% done | ETA(epoch): 530s + [Epoch 153][170/200] loss=0.618857 avg=0.655223 VRAM=38.6GiB | 30.8% done | ETA(epoch): 398s + [Epoch 153][180/200] loss=0.640007 avg=0.654538 VRAM=38.5GiB | 30.8% done | ETA(epoch): 265s + [Epoch 153][190/200] loss=0.585629 avg=0.653897 VRAM=38.6GiB | 30.8% done | ETA(epoch): 133s + [Epoch 153][200/200] loss=0.502580 avg=0.652055 VRAM=38.5GiB | 30.8% done | ETA(epoch): 0s + Train loss: 0.652055 (2650.3s) ETA: 17078min + Val loss: 0.698545 [t_0.0-0.2=0.9519 t_0.2-0.4=0.7581 t_0.4-0.6=0.6485 t_0.6-0.8=0.5627 t_0.8-1.0=0.5582] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 154/499 (31% done) --- + [Epoch 154][10/200] loss=0.645614 avg=0.637648 VRAM=38.6GiB | 30.8% done | ETA(epoch): 2517s + [Epoch 154][20/200] loss=0.638729 avg=0.643419 VRAM=38.5GiB | 30.8% done | ETA(epoch): 2383s + [Epoch 154][30/200] loss=0.690648 avg=0.645364 VRAM=38.6GiB | 30.8% done | ETA(epoch): 2250s + [Epoch 154][40/200] loss=0.713948 avg=0.656153 VRAM=38.5GiB | 30.8% done | ETA(epoch): 2118s + [Epoch 154][50/200] loss=0.793750 avg=0.657731 VRAM=38.6GiB | 30.9% done | ETA(epoch): 1985s + [Epoch 154][60/200] loss=0.690648 avg=0.661678 VRAM=38.5GiB | 30.9% done | ETA(epoch): 1853s + [Epoch 154][70/200] loss=0.736748 avg=0.659231 VRAM=38.6GiB | 30.9% done | ETA(epoch): 1721s + [Epoch 154][80/200] loss=0.717646 avg=0.662828 VRAM=38.5GiB | 30.9% done | ETA(epoch): 1589s + [Epoch 154][90/200] loss=0.708623 avg=0.664360 VRAM=38.6GiB | 30.9% done | ETA(epoch): 1456s + [Epoch 154][100/200] loss=0.572392 avg=0.666621 VRAM=38.5GiB | 30.9% done | ETA(epoch): 1324s + [Epoch 154][110/200] loss=0.742157 avg=0.666220 VRAM=38.6GiB | 30.9% done | ETA(epoch): 1191s + [Epoch 154][120/200] loss=0.601941 avg=0.665864 VRAM=38.5GiB | 30.9% done | ETA(epoch): 1059s + [Epoch 154][130/200] loss=0.634907 avg=0.666799 VRAM=38.6GiB | 30.9% done | ETA(epoch): 927s + [Epoch 154][140/200] loss=0.675842 avg=0.665598 VRAM=38.5GiB | 30.9% done | ETA(epoch): 794s + [Epoch 154][150/200] loss=0.813609 avg=0.665066 VRAM=38.6GiB | 30.9% done | ETA(epoch): 662s + [Epoch 154][160/200] loss=0.632078 avg=0.663430 VRAM=38.5GiB | 31.0% done | ETA(epoch): 529s + [Epoch 154][170/200] loss=0.648174 avg=0.663075 VRAM=38.6GiB | 31.0% done | ETA(epoch): 397s + [Epoch 154][180/200] loss=0.594450 avg=0.661023 VRAM=38.5GiB | 31.0% done | ETA(epoch): 265s + [Epoch 154][190/200] loss=0.655310 avg=0.661246 VRAM=38.6GiB | 31.0% done | ETA(epoch): 132s + [Epoch 154][200/200] loss=0.520348 avg=0.661260 VRAM=38.5GiB | 31.0% done | ETA(epoch): 0s + Train loss: 0.661260 (2649.5s) ETA: 17026min + Val loss: 0.700398 [t_0.0-0.2=0.9337 t_0.2-0.4=0.7441 t_0.4-0.6=0.6418 t_0.6-0.8=0.5733 t_0.8-1.0=0.5639] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 24.8/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 155/499 (31% done) --- + [Epoch 155][10/200] loss=0.622960 avg=0.669770 VRAM=38.6GiB | 31.0% done | ETA(epoch): 2514s + [Epoch 155][20/200] loss=0.555254 avg=0.662976 VRAM=38.5GiB | 31.0% done | ETA(epoch): 2381s + [Epoch 155][30/200] loss=0.703704 avg=0.649913 VRAM=38.6GiB | 31.0% done | ETA(epoch): 2250s + [Epoch 155][40/200] loss=0.767829 avg=0.663021 VRAM=38.5GiB | 31.0% done | ETA(epoch): 2117s + [Epoch 155][50/200] loss=0.782794 avg=0.662693 VRAM=38.6GiB | 31.1% done | ETA(epoch): 1985s + [Epoch 155][60/200] loss=0.592923 avg=0.659740 VRAM=38.5GiB | 31.1% done | ETA(epoch): 1853s + [Epoch 155][70/200] loss=0.685695 avg=0.662339 VRAM=38.6GiB | 31.1% done | ETA(epoch): 1727s + [Epoch 155][80/200] loss=0.545197 avg=0.655337 VRAM=38.5GiB | 31.1% done | ETA(epoch): 1593s + [Epoch 155][90/200] loss=0.597251 avg=0.655373 VRAM=38.6GiB | 31.1% done | ETA(epoch): 1460s + [Epoch 155][100/200] loss=0.642981 avg=0.657555 VRAM=38.5GiB | 31.1% done | ETA(epoch): 1327s + [Epoch 155][110/200] loss=0.630969 avg=0.656140 VRAM=38.6GiB | 31.1% done | ETA(epoch): 1194s + [Epoch 155][120/200] loss=0.708949 avg=0.657869 VRAM=38.5GiB | 31.1% done | ETA(epoch): 1061s + [Epoch 155][130/200] loss=0.623600 avg=0.653516 VRAM=38.6GiB | 31.1% done | ETA(epoch): 929s + [Epoch 155][140/200] loss=0.652880 avg=0.652835 VRAM=38.5GiB | 31.1% done | ETA(epoch): 796s + [Epoch 155][150/200] loss=0.686738 avg=0.654568 VRAM=38.6GiB | 31.1% done | ETA(epoch): 663s + [Epoch 155][160/200] loss=0.699423 avg=0.657158 VRAM=38.5GiB | 31.2% done | ETA(epoch): 530s + [Epoch 155][170/200] loss=0.731943 avg=0.660759 VRAM=38.6GiB | 31.2% done | ETA(epoch): 398s + [Epoch 155][180/200] loss=0.634769 avg=0.658858 VRAM=38.5GiB | 31.2% done | ETA(epoch): 265s + [Epoch 155][190/200] loss=0.693442 avg=0.660737 VRAM=38.6GiB | 31.2% done | ETA(epoch): 133s + [Epoch 155][200/200] loss=0.586603 avg=0.659743 VRAM=38.5GiB | 31.2% done | ETA(epoch): 0s + Train loss: 0.659743 (2651.3s) ETA: 16974min + Val loss: 0.692340 [t_0.0-0.2=0.9511 t_0.2-0.4=0.7472 t_0.4-0.6=0.6397 t_0.6-0.8=0.5625 t_0.8-1.0=0.5712] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 156/499 (31% done) --- + [Epoch 156][10/200] loss=0.535489 avg=0.683908 VRAM=38.6GiB | 31.2% done | ETA(epoch): 2514s + [Epoch 156][20/200] loss=0.731898 avg=0.678516 VRAM=38.5GiB | 31.2% done | ETA(epoch): 2383s + [Epoch 156][30/200] loss=0.619906 avg=0.685633 VRAM=38.6GiB | 31.2% done | ETA(epoch): 2265s + [Epoch 156][40/200] loss=0.765548 avg=0.688844 VRAM=38.5GiB | 31.2% done | ETA(epoch): 2128s + [Epoch 156][50/200] loss=0.616543 avg=0.677808 VRAM=38.6GiB | 31.2% done | ETA(epoch): 1993s + [Epoch 156][60/200] loss=0.587897 avg=0.675264 VRAM=38.5GiB | 31.3% done | ETA(epoch): 1859s + [Epoch 156][70/200] loss=0.621270 avg=0.671453 VRAM=38.6GiB | 31.3% done | ETA(epoch): 1725s + [Epoch 156][80/200] loss=0.720971 avg=0.667682 VRAM=38.5GiB | 31.3% done | ETA(epoch): 1592s + [Epoch 156][90/200] loss=0.695916 avg=0.665780 VRAM=38.6GiB | 31.3% done | ETA(epoch): 1459s + [Epoch 156][100/200] loss=0.754808 avg=0.667166 VRAM=38.5GiB | 31.3% done | ETA(epoch): 1326s + [Epoch 156][110/200] loss=0.794957 avg=0.671735 VRAM=38.6GiB | 31.3% done | ETA(epoch): 1194s + [Epoch 156][120/200] loss=0.590109 avg=0.670135 VRAM=38.5GiB | 31.3% done | ETA(epoch): 1065s + [Epoch 156][130/200] loss=0.734741 avg=0.671561 VRAM=38.6GiB | 31.3% done | ETA(epoch): 931s + [Epoch 156][140/200] loss=0.700888 avg=0.669352 VRAM=38.5GiB | 31.3% done | ETA(epoch): 798s + [Epoch 156][150/200] loss=0.636211 avg=0.669406 VRAM=38.6GiB | 31.4% done | ETA(epoch): 665s + [Epoch 156][160/200] loss=0.667578 avg=0.669137 VRAM=38.5GiB | 31.4% done | ETA(epoch): 532s + [Epoch 156][170/200] loss=0.672688 avg=0.668013 VRAM=38.6GiB | 31.4% done | ETA(epoch): 399s + [Epoch 156][180/200] loss=0.521770 avg=0.666442 VRAM=38.5GiB | 31.4% done | ETA(epoch): 266s + [Epoch 156][190/200] loss=0.629378 avg=0.667261 VRAM=38.6GiB | 31.4% done | ETA(epoch): 133s + [Epoch 156][200/200] loss=0.674038 avg=0.665658 VRAM=38.5GiB | 31.4% done | ETA(epoch): 0s + Train loss: 0.665658 (2656.2s) ETA: 16923min + Val loss: 0.683676 [t_0.0-0.2=0.9140 t_0.2-0.4=0.7407 t_0.4-0.6=0.6393 t_0.6-0.8=0.5716 t_0.8-1.0=0.5564] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 157/499 (31% done) --- + [Epoch 157][10/200] loss=0.647120 avg=0.669366 VRAM=38.6GiB | 31.4% done | ETA(epoch): 2513s + [Epoch 157][20/200] loss=0.647565 avg=0.685939 VRAM=38.5GiB | 31.4% done | ETA(epoch): 2381s + [Epoch 157][30/200] loss=0.754200 avg=0.671774 VRAM=38.6GiB | 31.4% done | ETA(epoch): 2249s + [Epoch 157][40/200] loss=0.795388 avg=0.658516 VRAM=38.5GiB | 31.4% done | ETA(epoch): 2118s + [Epoch 157][50/200] loss=0.618985 avg=0.660158 VRAM=38.6GiB | 31.4% done | ETA(epoch): 1985s + [Epoch 157][60/200] loss=0.779745 avg=0.660073 VRAM=38.5GiB | 31.5% done | ETA(epoch): 1853s + [Epoch 157][70/200] loss=0.831708 avg=0.665452 VRAM=38.6GiB | 31.5% done | ETA(epoch): 1720s + [Epoch 157][80/200] loss=0.703578 avg=0.668578 VRAM=38.5GiB | 31.5% done | ETA(epoch): 1593s + [Epoch 157][90/200] loss=0.630224 avg=0.664651 VRAM=38.6GiB | 31.5% done | ETA(epoch): 1460s + [Epoch 157][100/200] loss=0.785327 avg=0.666963 VRAM=38.5GiB | 31.5% done | ETA(epoch): 1327s + [Epoch 157][110/200] loss=0.800396 avg=0.668817 VRAM=38.6GiB | 31.5% done | ETA(epoch): 1194s + [Epoch 157][120/200] loss=0.647805 avg=0.672258 VRAM=38.5GiB | 31.5% done | ETA(epoch): 1061s + [Epoch 157][130/200] loss=0.877605 avg=0.670955 VRAM=38.6GiB | 31.5% done | ETA(epoch): 928s + [Epoch 157][140/200] loss=0.628667 avg=0.672099 VRAM=38.5GiB | 31.5% done | ETA(epoch): 795s + [Epoch 157][150/200] loss=0.613749 avg=0.669314 VRAM=38.6GiB | 31.6% done | ETA(epoch): 663s + [Epoch 157][160/200] loss=0.597609 avg=0.665016 VRAM=38.5GiB | 31.6% done | ETA(epoch): 530s + [Epoch 157][170/200] loss=0.622122 avg=0.663298 VRAM=38.6GiB | 31.6% done | ETA(epoch): 398s + [Epoch 157][180/200] loss=0.575380 avg=0.661870 VRAM=38.5GiB | 31.6% done | ETA(epoch): 265s + [Epoch 157][190/200] loss=0.577316 avg=0.661726 VRAM=38.6GiB | 31.6% done | ETA(epoch): 133s + [Epoch 157][200/200] loss=0.704284 avg=0.661346 VRAM=38.5GiB | 31.6% done | ETA(epoch): 0s + Train loss: 0.661346 (2653.2s) ETA: 16871min + Val loss: 0.698445 [t_0.0-0.2=0.9575 t_0.2-0.4=0.7424 t_0.4-0.6=0.6477 t_0.6-0.8=0.5491 t_0.8-1.0=0.5624] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 158/499 (32% done) --- + [Epoch 158][10/200] loss=0.562266 avg=0.621686 VRAM=38.6GiB | 31.6% done | ETA(epoch): 2520s + [Epoch 158][20/200] loss=0.633899 avg=0.622018 VRAM=38.5GiB | 31.6% done | ETA(epoch): 2384s + [Epoch 158][30/200] loss=0.694862 avg=0.631549 VRAM=38.6GiB | 31.6% done | ETA(epoch): 2252s + [Epoch 158][40/200] loss=0.704676 avg=0.641560 VRAM=38.5GiB | 31.6% done | ETA(epoch): 2119s + [Epoch 158][50/200] loss=0.600930 avg=0.644288 VRAM=38.6GiB | 31.6% done | ETA(epoch): 1987s + [Epoch 158][60/200] loss=0.610408 avg=0.641749 VRAM=38.5GiB | 31.7% done | ETA(epoch): 1854s + [Epoch 158][70/200] loss=0.704916 avg=0.643686 VRAM=38.6GiB | 31.7% done | ETA(epoch): 1721s + [Epoch 158][80/200] loss=0.698117 avg=0.643253 VRAM=38.5GiB | 31.7% done | ETA(epoch): 1589s + [Epoch 158][90/200] loss=0.617305 avg=0.641856 VRAM=38.6GiB | 31.7% done | ETA(epoch): 1456s + [Epoch 158][100/200] loss=0.756230 avg=0.645208 VRAM=38.5GiB | 31.7% done | ETA(epoch): 1324s + [Epoch 158][110/200] loss=0.600274 avg=0.642723 VRAM=38.6GiB | 31.7% done | ETA(epoch): 1191s + [Epoch 158][120/200] loss=0.657405 avg=0.643503 VRAM=38.5GiB | 31.7% done | ETA(epoch): 1059s + [Epoch 158][130/200] loss=0.714192 avg=0.644293 VRAM=38.6GiB | 31.7% done | ETA(epoch): 927s + [Epoch 158][140/200] loss=0.639492 avg=0.645616 VRAM=38.5GiB | 31.7% done | ETA(epoch): 795s + [Epoch 158][150/200] loss=0.718075 avg=0.649322 VRAM=38.6GiB | 31.8% done | ETA(epoch): 663s + [Epoch 158][160/200] loss=0.553054 avg=0.648073 VRAM=38.5GiB | 31.8% done | ETA(epoch): 530s + [Epoch 158][170/200] loss=0.650156 avg=0.646395 VRAM=38.6GiB | 31.8% done | ETA(epoch): 398s + [Epoch 158][180/200] loss=0.669178 avg=0.647282 VRAM=38.5GiB | 31.8% done | ETA(epoch): 265s + [Epoch 158][190/200] loss=0.573522 avg=0.647618 VRAM=38.6GiB | 31.8% done | ETA(epoch): 133s + [Epoch 158][200/200] loss=0.755523 avg=0.649270 VRAM=38.5GiB | 31.8% done | ETA(epoch): 0s + Train loss: 0.649270 (2650.3s) ETA: 16820min + Val loss: 0.686103 [t_0.0-0.2=0.9158 t_0.2-0.4=0.7478 t_0.4-0.6=0.6337 t_0.6-0.8=0.5616 t_0.8-1.0=0.5681] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 159/499 (32% done) --- + [Epoch 159][10/200] loss=0.623430 avg=0.653870 VRAM=38.6GiB | 31.8% done | ETA(epoch): 2521s + [Epoch 159][20/200] loss=0.608984 avg=0.655278 VRAM=38.5GiB | 31.8% done | ETA(epoch): 2385s + [Epoch 159][30/200] loss=0.623585 avg=0.660921 VRAM=38.6GiB | 31.8% done | ETA(epoch): 2251s + [Epoch 159][40/200] loss=0.655665 avg=0.670046 VRAM=38.5GiB | 31.8% done | ETA(epoch): 2119s + [Epoch 159][50/200] loss=0.659508 avg=0.669558 VRAM=38.6GiB | 31.9% done | ETA(epoch): 1986s + [Epoch 159][60/200] loss=0.653741 avg=0.674117 VRAM=38.5GiB | 31.9% done | ETA(epoch): 1854s + [Epoch 159][70/200] loss=0.555754 avg=0.671242 VRAM=38.6GiB | 31.9% done | ETA(epoch): 1721s + [Epoch 159][80/200] loss=0.690567 avg=0.668431 VRAM=38.5GiB | 31.9% done | ETA(epoch): 1588s + [Epoch 159][90/200] loss=0.654688 avg=0.674001 VRAM=38.6GiB | 31.9% done | ETA(epoch): 1456s + [Epoch 159][100/200] loss=0.606490 avg=0.668889 VRAM=38.5GiB | 31.9% done | ETA(epoch): 1324s + [Epoch 159][110/200] loss=0.613184 avg=0.665138 VRAM=38.6GiB | 31.9% done | ETA(epoch): 1191s + [Epoch 159][120/200] loss=0.562937 avg=0.665535 VRAM=38.5GiB | 31.9% done | ETA(epoch): 1059s + [Epoch 159][130/200] loss=0.861424 avg=0.665861 VRAM=38.6GiB | 31.9% done | ETA(epoch): 927s + [Epoch 159][140/200] loss=0.611732 avg=0.661283 VRAM=38.5GiB | 31.9% done | ETA(epoch): 794s + [Epoch 159][150/200] loss=0.565805 avg=0.659194 VRAM=38.6GiB | 31.9% done | ETA(epoch): 662s + [Epoch 159][160/200] loss=0.698129 avg=0.659312 VRAM=38.5GiB | 32.0% done | ETA(epoch): 529s + [Epoch 159][170/200] loss=0.631053 avg=0.659107 VRAM=38.6GiB | 32.0% done | ETA(epoch): 397s + [Epoch 159][180/200] loss=0.611515 avg=0.659548 VRAM=38.5GiB | 32.0% done | ETA(epoch): 265s + [Epoch 159][190/200] loss=0.616445 avg=0.659624 VRAM=38.6GiB | 32.0% done | ETA(epoch): 132s + [Epoch 159][200/200] loss=0.751876 avg=0.660579 VRAM=38.5GiB | 32.0% done | ETA(epoch): 0s + Train loss: 0.660579 (2647.0s) ETA: 16768min + Val loss: 0.704039 [t_0.0-0.2=0.9402 t_0.2-0.4=0.7490 t_0.4-0.6=0.6399 t_0.6-0.8=0.5641 t_0.8-1.0=0.5527] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 160/499 (32% done) --- + [Epoch 160][10/200] loss=0.678987 avg=0.637524 VRAM=38.6GiB | 32.0% done | ETA(epoch): 2518s + [Epoch 160][20/200] loss=0.631801 avg=0.630935 VRAM=38.5GiB | 32.0% done | ETA(epoch): 2383s + [Epoch 160][30/200] loss=0.568279 avg=0.645921 VRAM=38.6GiB | 32.0% done | ETA(epoch): 2250s + [Epoch 160][40/200] loss=0.698911 avg=0.640600 VRAM=38.5GiB | 32.0% done | ETA(epoch): 2118s + [Epoch 160][50/200] loss=0.604621 avg=0.637697 VRAM=38.6GiB | 32.0% done | ETA(epoch): 1986s + [Epoch 160][60/200] loss=0.642150 avg=0.644436 VRAM=38.5GiB | 32.1% done | ETA(epoch): 1862s + [Epoch 160][70/200] loss=0.756739 avg=0.648829 VRAM=38.6GiB | 32.1% done | ETA(epoch): 1728s + [Epoch 160][80/200] loss=0.570005 avg=0.656172 VRAM=38.5GiB | 32.1% done | ETA(epoch): 1594s + [Epoch 160][90/200] loss=0.737571 avg=0.657278 VRAM=38.6GiB | 32.1% done | ETA(epoch): 1461s + [Epoch 160][100/200] loss=0.603572 avg=0.655940 VRAM=38.5GiB | 32.1% done | ETA(epoch): 1328s + [Epoch 160][110/200] loss=0.530392 avg=0.656497 VRAM=38.6GiB | 32.1% done | ETA(epoch): 1195s + [Epoch 160][120/200] loss=0.681328 avg=0.656776 VRAM=38.5GiB | 32.1% done | ETA(epoch): 1062s + [Epoch 160][130/200] loss=0.584894 avg=0.656090 VRAM=38.6GiB | 32.1% done | ETA(epoch): 929s + [Epoch 160][140/200] loss=0.570970 avg=0.658648 VRAM=38.5GiB | 32.1% done | ETA(epoch): 796s + [Epoch 160][150/200] loss=0.667911 avg=0.656519 VRAM=38.6GiB | 32.1% done | ETA(epoch): 663s + [Epoch 160][160/200] loss=0.713692 avg=0.655781 VRAM=38.5GiB | 32.2% done | ETA(epoch): 530s + [Epoch 160][170/200] loss=0.601220 avg=0.654243 VRAM=38.6GiB | 32.2% done | ETA(epoch): 398s + [Epoch 160][180/200] loss=0.642345 avg=0.654245 VRAM=38.5GiB | 32.2% done | ETA(epoch): 265s + [Epoch 160][190/200] loss=0.515356 avg=0.655576 VRAM=38.6GiB | 32.2% done | ETA(epoch): 133s + [Epoch 160][200/200] loss=0.650326 avg=0.652895 VRAM=38.5GiB | 32.2% done | ETA(epoch): 0s + Train loss: 0.652895 (2651.4s) ETA: 16717min + Val loss: 0.693745 [t_0.0-0.2=0.9378 t_0.2-0.4=0.7483 t_0.4-0.6=0.6429 t_0.6-0.8=0.5581 t_0.8-1.0=0.5656] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 161/499 (32% done) --- + [Epoch 161][10/200] loss=0.675440 avg=0.670276 VRAM=38.6GiB | 32.2% done | ETA(epoch): 2511s + [Epoch 161][20/200] loss=0.610829 avg=0.663116 VRAM=38.5GiB | 32.2% done | ETA(epoch): 2379s + [Epoch 161][30/200] loss=0.499930 avg=0.666086 VRAM=38.6GiB | 32.2% done | ETA(epoch): 2289s + [Epoch 161][40/200] loss=0.682871 avg=0.659719 VRAM=38.5GiB | 32.2% done | ETA(epoch): 2145s + [Epoch 161][50/200] loss=0.565512 avg=0.652951 VRAM=38.6GiB | 32.2% done | ETA(epoch): 2007s + [Epoch 161][60/200] loss=0.685918 avg=0.655641 VRAM=38.5GiB | 32.3% done | ETA(epoch): 1869s + [Epoch 161][70/200] loss=0.798806 avg=0.655376 VRAM=38.6GiB | 32.3% done | ETA(epoch): 1734s + [Epoch 161][80/200] loss=0.590274 avg=0.660297 VRAM=38.5GiB | 32.3% done | ETA(epoch): 1599s + [Epoch 161][90/200] loss=0.647612 avg=0.657529 VRAM=38.6GiB | 32.3% done | ETA(epoch): 1465s + [Epoch 161][100/200] loss=0.782212 avg=0.659530 VRAM=38.5GiB | 32.3% done | ETA(epoch): 1331s + [Epoch 161][110/200] loss=0.666796 avg=0.660201 VRAM=38.6GiB | 32.3% done | ETA(epoch): 1197s + [Epoch 161][120/200] loss=0.648002 avg=0.659444 VRAM=38.5GiB | 32.3% done | ETA(epoch): 1068s + [Epoch 161][130/200] loss=0.599168 avg=0.657045 VRAM=38.6GiB | 32.3% done | ETA(epoch): 934s + [Epoch 161][140/200] loss=0.587882 avg=0.657690 VRAM=38.5GiB | 32.3% done | ETA(epoch): 800s + [Epoch 161][150/200] loss=0.662886 avg=0.657531 VRAM=38.6GiB | 32.4% done | ETA(epoch): 666s + [Epoch 161][160/200] loss=0.600161 avg=0.656511 VRAM=38.5GiB | 32.4% done | ETA(epoch): 533s + [Epoch 161][170/200] loss=0.735394 avg=0.657891 VRAM=38.6GiB | 32.4% done | ETA(epoch): 399s + [Epoch 161][180/200] loss=0.644639 avg=0.657368 VRAM=38.5GiB | 32.4% done | ETA(epoch): 266s + [Epoch 161][190/200] loss=0.622782 avg=0.658117 VRAM=38.6GiB | 32.4% done | ETA(epoch): 133s + [Epoch 161][200/200] loss=0.631162 avg=0.656708 VRAM=38.5GiB | 32.4% done | ETA(epoch): 0s + Train loss: 0.656708 (2660.9s) ETA: 16666min + Val loss: 0.679849 [t_0.0-0.2=0.9208 t_0.2-0.4=0.7441 t_0.4-0.6=0.6335 t_0.6-0.8=0.5577 t_0.8-1.0=0.5722] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 162/499 (32% done) --- + [Epoch 162][10/200] loss=0.601631 avg=0.627636 VRAM=38.6GiB | 32.4% done | ETA(epoch): 2511s + [Epoch 162][20/200] loss=0.606355 avg=0.619172 VRAM=38.5GiB | 32.4% done | ETA(epoch): 2380s + [Epoch 162][30/200] loss=0.659832 avg=0.626621 VRAM=38.6GiB | 32.4% done | ETA(epoch): 2250s + [Epoch 162][40/200] loss=0.664400 avg=0.648607 VRAM=38.5GiB | 32.4% done | ETA(epoch): 2117s + [Epoch 162][50/200] loss=0.588361 avg=0.650340 VRAM=38.6GiB | 32.5% done | ETA(epoch): 1986s + [Epoch 162][60/200] loss=0.700845 avg=0.649025 VRAM=38.5GiB | 32.5% done | ETA(epoch): 1853s + [Epoch 162][70/200] loss=0.631526 avg=0.651056 VRAM=38.6GiB | 32.5% done | ETA(epoch): 1721s + [Epoch 162][80/200] loss=0.639004 avg=0.648720 VRAM=38.5GiB | 32.5% done | ETA(epoch): 1593s + [Epoch 162][90/200] loss=0.731534 avg=0.655510 VRAM=38.6GiB | 32.5% done | ETA(epoch): 1460s + [Epoch 162][100/200] loss=0.668531 avg=0.656811 VRAM=38.5GiB | 32.5% done | ETA(epoch): 1327s + [Epoch 162][110/200] loss=0.653992 avg=0.653785 VRAM=38.6GiB | 32.5% done | ETA(epoch): 1194s + [Epoch 162][120/200] loss=0.685207 avg=0.656381 VRAM=38.5GiB | 32.5% done | ETA(epoch): 1061s + [Epoch 162][130/200] loss=0.627638 avg=0.655884 VRAM=38.6GiB | 32.5% done | ETA(epoch): 928s + [Epoch 162][140/200] loss=0.659458 avg=0.655339 VRAM=38.5GiB | 32.5% done | ETA(epoch): 796s + [Epoch 162][150/200] loss=0.652028 avg=0.654124 VRAM=38.6GiB | 32.6% done | ETA(epoch): 663s + [Epoch 162][160/200] loss=0.639775 avg=0.653272 VRAM=38.5GiB | 32.6% done | ETA(epoch): 530s + [Epoch 162][170/200] loss=0.567545 avg=0.653592 VRAM=38.6GiB | 32.6% done | ETA(epoch): 398s + [Epoch 162][180/200] loss=0.685611 avg=0.654263 VRAM=38.5GiB | 32.6% done | ETA(epoch): 265s + [Epoch 162][190/200] loss=0.659770 avg=0.652903 VRAM=38.6GiB | 32.6% done | ETA(epoch): 133s + [Epoch 162][200/200] loss=0.685850 avg=0.653722 VRAM=38.5GiB | 32.6% done | ETA(epoch): 0s + Train loss: 0.653722 (2650.9s) ETA: 16615min + Val loss: 0.683583 [t_0.0-0.2=0.9207 t_0.2-0.4=0.7701 t_0.4-0.6=0.6430 t_0.6-0.8=0.5642 t_0.8-1.0=0.5658] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 163/499 (33% done) --- + [Epoch 163][10/200] loss=0.606610 avg=0.649735 VRAM=38.6GiB | 32.6% done | ETA(epoch): 2521s + [Epoch 163][20/200] loss=0.736354 avg=0.657413 VRAM=38.5GiB | 32.6% done | ETA(epoch): 2385s + [Epoch 163][30/200] loss=0.612778 avg=0.651576 VRAM=38.6GiB | 32.6% done | ETA(epoch): 2251s + [Epoch 163][40/200] loss=0.742916 avg=0.654632 VRAM=38.5GiB | 32.6% done | ETA(epoch): 2132s + [Epoch 163][50/200] loss=0.696179 avg=0.657962 VRAM=38.6GiB | 32.6% done | ETA(epoch): 1997s + [Epoch 163][60/200] loss=0.518754 avg=0.659927 VRAM=38.5GiB | 32.7% done | ETA(epoch): 1862s + [Epoch 163][70/200] loss=0.691069 avg=0.655330 VRAM=38.6GiB | 32.7% done | ETA(epoch): 1728s + [Epoch 163][80/200] loss=0.616694 avg=0.657218 VRAM=38.5GiB | 32.7% done | ETA(epoch): 1594s + [Epoch 163][90/200] loss=0.662388 avg=0.659938 VRAM=38.6GiB | 32.7% done | ETA(epoch): 1460s + [Epoch 163][100/200] loss=0.622996 avg=0.656407 VRAM=38.5GiB | 32.7% done | ETA(epoch): 1327s + [Epoch 163][110/200] loss=0.859927 avg=0.655833 VRAM=38.6GiB | 32.7% done | ETA(epoch): 1194s + [Epoch 163][120/200] loss=0.692881 avg=0.655077 VRAM=38.5GiB | 32.7% done | ETA(epoch): 1061s + [Epoch 163][130/200] loss=0.826493 avg=0.654016 VRAM=38.6GiB | 32.7% done | ETA(epoch): 930s + [Epoch 163][140/200] loss=0.605603 avg=0.652408 VRAM=38.5GiB | 32.7% done | ETA(epoch): 797s + [Epoch 163][150/200] loss=0.555980 avg=0.653292 VRAM=38.6GiB | 32.8% done | ETA(epoch): 664s + [Epoch 163][160/200] loss=0.779924 avg=0.652507 VRAM=38.5GiB | 32.8% done | ETA(epoch): 531s + [Epoch 163][170/200] loss=0.579848 avg=0.653150 VRAM=38.6GiB | 32.8% done | ETA(epoch): 398s + [Epoch 163][180/200] loss=0.648262 avg=0.653252 VRAM=38.5GiB | 32.8% done | ETA(epoch): 265s + [Epoch 163][190/200] loss=0.767255 avg=0.652950 VRAM=38.6GiB | 32.8% done | ETA(epoch): 133s + [Epoch 163][200/200] loss=0.566462 avg=0.654470 VRAM=38.5GiB | 32.8% done | ETA(epoch): 0s + Train loss: 0.654470 (2653.9s) ETA: 16564min + Val loss: 0.687258 [t_0.0-0.2=0.9335 t_0.2-0.4=0.7546 t_0.4-0.6=0.6310 t_0.6-0.8=0.5523 t_0.8-1.0=0.5774] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 164/499 (33% done) --- + [Epoch 164][10/200] loss=0.922432 avg=0.667310 VRAM=38.6GiB | 32.8% done | ETA(epoch): 2589s + [Epoch 164][20/200] loss=0.732395 avg=0.650162 VRAM=38.5GiB | 32.8% done | ETA(epoch): 2417s + [Epoch 164][30/200] loss=0.636624 avg=0.654415 VRAM=38.6GiB | 32.8% done | ETA(epoch): 2272s + [Epoch 164][40/200] loss=0.637823 avg=0.648557 VRAM=38.5GiB | 32.8% done | ETA(epoch): 2134s + [Epoch 164][50/200] loss=0.678424 avg=0.644954 VRAM=38.6GiB | 32.9% done | ETA(epoch): 1997s + [Epoch 164][60/200] loss=0.608986 avg=0.648184 VRAM=38.5GiB | 32.9% done | ETA(epoch): 1863s + [Epoch 164][70/200] loss=0.544333 avg=0.652092 VRAM=38.6GiB | 32.9% done | ETA(epoch): 1728s + [Epoch 164][80/200] loss=0.667213 avg=0.653239 VRAM=38.5GiB | 32.9% done | ETA(epoch): 1595s + [Epoch 164][90/200] loss=0.713235 avg=0.656622 VRAM=38.6GiB | 32.9% done | ETA(epoch): 1461s + [Epoch 164][100/200] loss=0.574330 avg=0.660397 VRAM=38.5GiB | 32.9% done | ETA(epoch): 1328s + [Epoch 164][110/200] loss=0.592270 avg=0.659168 VRAM=38.6GiB | 32.9% done | ETA(epoch): 1195s + [Epoch 164][120/200] loss=0.617987 avg=0.657590 VRAM=38.5GiB | 32.9% done | ETA(epoch): 1062s + [Epoch 164][130/200] loss=0.659565 avg=0.655021 VRAM=38.6GiB | 32.9% done | ETA(epoch): 929s + [Epoch 164][140/200] loss=0.606904 avg=0.655133 VRAM=38.5GiB | 32.9% done | ETA(epoch): 796s + [Epoch 164][150/200] loss=0.635507 avg=0.656516 VRAM=38.6GiB | 33.0% done | ETA(epoch): 663s + [Epoch 164][160/200] loss=0.631363 avg=0.656210 VRAM=38.5GiB | 33.0% done | ETA(epoch): 531s + [Epoch 164][170/200] loss=0.617084 avg=0.654552 VRAM=38.6GiB | 33.0% done | ETA(epoch): 398s + [Epoch 164][180/200] loss=0.656400 avg=0.655425 VRAM=38.5GiB | 33.0% done | ETA(epoch): 265s + [Epoch 164][190/200] loss=0.692276 avg=0.657046 VRAM=38.6GiB | 33.0% done | ETA(epoch): 133s + [Epoch 164][200/200] loss=0.738919 avg=0.658003 VRAM=38.5GiB | 33.0% done | ETA(epoch): 0s + Train loss: 0.658003 (2654.6s) ETA: 16512min + Val loss: 0.687882 [t_0.0-0.2=0.9474 t_0.2-0.4=0.7509 t_0.4-0.6=0.6497 t_0.6-0.8=0.5575 t_0.8-1.0=0.5679] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 165/499 (33% done) --- + [Epoch 165][10/200] loss=0.759042 avg=0.609207 VRAM=38.6GiB | 33.0% done | ETA(epoch): 2515s + [Epoch 165][20/200] loss=0.646856 avg=0.621650 VRAM=38.5GiB | 33.0% done | ETA(epoch): 2382s + [Epoch 165][30/200] loss=0.645651 avg=0.647896 VRAM=38.6GiB | 33.0% done | ETA(epoch): 2250s + [Epoch 165][40/200] loss=0.768542 avg=0.650174 VRAM=38.5GiB | 33.0% done | ETA(epoch): 2119s + [Epoch 165][50/200] loss=0.619280 avg=0.650523 VRAM=38.6GiB | 33.1% done | ETA(epoch): 1986s + [Epoch 165][60/200] loss=0.644810 avg=0.649426 VRAM=38.5GiB | 33.1% done | ETA(epoch): 1860s + [Epoch 165][70/200] loss=0.621141 avg=0.639163 VRAM=38.6GiB | 33.1% done | ETA(epoch): 1727s + [Epoch 165][80/200] loss=0.623784 avg=0.643280 VRAM=38.5GiB | 33.1% done | ETA(epoch): 1593s + [Epoch 165][90/200] loss=0.632207 avg=0.644238 VRAM=38.6GiB | 33.1% done | ETA(epoch): 1460s + [Epoch 165][100/200] loss=0.664129 avg=0.642191 VRAM=38.5GiB | 33.1% done | ETA(epoch): 1327s + [Epoch 165][110/200] loss=0.694047 avg=0.644096 VRAM=38.6GiB | 33.1% done | ETA(epoch): 1194s + [Epoch 165][120/200] loss=0.615735 avg=0.645047 VRAM=38.5GiB | 33.1% done | ETA(epoch): 1061s + [Epoch 165][130/200] loss=0.616954 avg=0.646447 VRAM=38.6GiB | 33.1% done | ETA(epoch): 928s + [Epoch 165][140/200] loss=0.584419 avg=0.647843 VRAM=38.5GiB | 33.1% done | ETA(epoch): 795s + [Epoch 165][150/200] loss=0.674081 avg=0.647492 VRAM=38.6GiB | 33.1% done | ETA(epoch): 663s + [Epoch 165][160/200] loss=0.529478 avg=0.647907 VRAM=38.5GiB | 33.2% done | ETA(epoch): 530s + [Epoch 165][170/200] loss=0.755600 avg=0.647361 VRAM=38.6GiB | 33.2% done | ETA(epoch): 398s + [Epoch 165][180/200] loss=0.590783 avg=0.648483 VRAM=38.5GiB | 33.2% done | ETA(epoch): 265s + [Epoch 165][190/200] loss=0.625685 avg=0.650963 VRAM=38.6GiB | 33.2% done | ETA(epoch): 133s + [Epoch 165][200/200] loss=0.749198 avg=0.652588 VRAM=38.5GiB | 33.2% done | ETA(epoch): 0s + Train loss: 0.652588 (2650.1s) ETA: 16461min + Val loss: 0.691915 [t_0.0-0.2=0.9435 t_0.2-0.4=0.7435 t_0.4-0.6=0.6309 t_0.6-0.8=0.5729 t_0.8-1.0=0.5645] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 166/499 (33% done) --- + [Epoch 166][10/200] loss=0.712270 avg=0.662505 VRAM=38.6GiB | 33.2% done | ETA(epoch): 2517s + [Epoch 166][20/200] loss=0.769279 avg=0.672088 VRAM=38.5GiB | 33.2% done | ETA(epoch): 2413s + [Epoch 166][30/200] loss=0.678568 avg=0.663854 VRAM=38.6GiB | 33.2% done | ETA(epoch): 2270s + [Epoch 166][40/200] loss=0.602772 avg=0.668706 VRAM=38.5GiB | 33.2% done | ETA(epoch): 2134s + [Epoch 166][50/200] loss=0.623869 avg=0.661281 VRAM=38.6GiB | 33.2% done | ETA(epoch): 1997s + [Epoch 166][60/200] loss=0.595121 avg=0.654113 VRAM=38.5GiB | 33.3% done | ETA(epoch): 1862s + [Epoch 166][70/200] loss=0.569918 avg=0.654735 VRAM=38.6GiB | 33.3% done | ETA(epoch): 1728s + [Epoch 166][80/200] loss=0.624676 avg=0.655791 VRAM=38.5GiB | 33.3% done | ETA(epoch): 1595s + [Epoch 166][90/200] loss=0.655951 avg=0.657633 VRAM=38.6GiB | 33.3% done | ETA(epoch): 1461s + [Epoch 166][100/200] loss=0.587422 avg=0.660784 VRAM=38.5GiB | 33.3% done | ETA(epoch): 1328s + [Epoch 166][110/200] loss=0.613677 avg=0.662200 VRAM=38.6GiB | 33.3% done | ETA(epoch): 1197s + [Epoch 166][120/200] loss=0.699882 avg=0.663778 VRAM=38.5GiB | 33.3% done | ETA(epoch): 1064s + [Epoch 166][130/200] loss=0.744110 avg=0.663160 VRAM=38.6GiB | 33.3% done | ETA(epoch): 931s + [Epoch 166][140/200] loss=0.657500 avg=0.659619 VRAM=38.5GiB | 33.3% done | ETA(epoch): 797s + [Epoch 166][150/200] loss=0.602479 avg=0.658800 VRAM=38.6GiB | 33.4% done | ETA(epoch): 664s + [Epoch 166][160/200] loss=0.675789 avg=0.660029 VRAM=38.5GiB | 33.4% done | ETA(epoch): 531s + [Epoch 166][170/200] loss=0.500428 avg=0.659071 VRAM=38.6GiB | 33.4% done | ETA(epoch): 398s + [Epoch 166][180/200] loss=0.568769 avg=0.658364 VRAM=38.5GiB | 33.4% done | ETA(epoch): 266s + [Epoch 166][190/200] loss=0.579928 avg=0.657308 VRAM=38.6GiB | 33.4% done | ETA(epoch): 133s + [Epoch 166][200/200] loss=0.724671 avg=0.658193 VRAM=38.5GiB | 33.4% done | ETA(epoch): 0s + Train loss: 0.658193 (2655.1s) ETA: 16410min + Val loss: 0.688898 [t_0.0-0.2=0.9363 t_0.2-0.4=0.7505 t_0.4-0.6=0.6380 t_0.6-0.8=0.5555 t_0.8-1.0=0.5655] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 167/499 (33% done) --- + [Epoch 167][10/200] loss=0.620918 avg=0.647644 VRAM=38.6GiB | 33.4% done | ETA(epoch): 2514s + [Epoch 167][20/200] loss=0.579578 avg=0.647003 VRAM=38.5GiB | 33.4% done | ETA(epoch): 2383s + [Epoch 167][30/200] loss=0.590052 avg=0.641412 VRAM=38.6GiB | 33.4% done | ETA(epoch): 2250s + [Epoch 167][40/200] loss=0.672892 avg=0.638976 VRAM=38.5GiB | 33.4% done | ETA(epoch): 2118s + [Epoch 167][50/200] loss=0.605438 avg=0.647763 VRAM=38.6GiB | 33.5% done | ETA(epoch): 1985s + [Epoch 167][60/200] loss=0.630206 avg=0.647511 VRAM=38.5GiB | 33.5% done | ETA(epoch): 1853s + [Epoch 167][70/200] loss=0.510568 avg=0.659352 VRAM=38.6GiB | 33.5% done | ETA(epoch): 1721s + [Epoch 167][80/200] loss=0.616622 avg=0.655104 VRAM=38.5GiB | 33.5% done | ETA(epoch): 1588s + [Epoch 167][90/200] loss=0.685462 avg=0.657819 VRAM=38.6GiB | 33.5% done | ETA(epoch): 1456s + [Epoch 167][100/200] loss=0.539219 avg=0.654212 VRAM=38.5GiB | 33.5% done | ETA(epoch): 1324s + [Epoch 167][110/200] loss=0.610506 avg=0.651582 VRAM=38.6GiB | 33.5% done | ETA(epoch): 1191s + [Epoch 167][120/200] loss=0.626671 avg=0.651267 VRAM=38.5GiB | 33.5% done | ETA(epoch): 1059s + [Epoch 167][130/200] loss=0.645306 avg=0.651445 VRAM=38.6GiB | 33.5% done | ETA(epoch): 927s + [Epoch 167][140/200] loss=0.586085 avg=0.650161 VRAM=38.5GiB | 33.5% done | ETA(epoch): 794s + [Epoch 167][150/200] loss=0.583296 avg=0.649732 VRAM=38.6GiB | 33.6% done | ETA(epoch): 662s + [Epoch 167][160/200] loss=0.626228 avg=0.649854 VRAM=38.5GiB | 33.6% done | ETA(epoch): 530s + [Epoch 167][170/200] loss=0.740247 avg=0.648504 VRAM=38.6GiB | 33.6% done | ETA(epoch): 398s + [Epoch 167][180/200] loss=0.703056 avg=0.649513 VRAM=38.5GiB | 33.6% done | ETA(epoch): 265s + [Epoch 167][190/200] loss=0.651950 avg=0.651429 VRAM=38.6GiB | 33.6% done | ETA(epoch): 133s + [Epoch 167][200/200] loss=0.711262 avg=0.650574 VRAM=38.5GiB | 33.6% done | ETA(epoch): 0s + Train loss: 0.650574 (2650.9s) ETA: 16359min + Val loss: 0.710227 [t_0.0-0.2=0.9151 t_0.2-0.4=0.7557 t_0.4-0.6=0.6584 t_0.6-0.8=0.5673 t_0.8-1.0=0.5691] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 168/499 (34% done) --- + [Epoch 168][10/200] loss=0.498181 avg=0.650612 VRAM=38.6GiB | 33.6% done | ETA(epoch): 2518s + [Epoch 168][20/200] loss=0.618141 avg=0.654755 VRAM=38.5GiB | 33.6% done | ETA(epoch): 2384s + [Epoch 168][30/200] loss=0.756063 avg=0.665485 VRAM=38.6GiB | 33.6% done | ETA(epoch): 2251s + [Epoch 168][40/200] loss=0.642346 avg=0.665442 VRAM=38.5GiB | 33.6% done | ETA(epoch): 2130s + [Epoch 168][50/200] loss=0.594750 avg=0.664583 VRAM=38.6GiB | 33.7% done | ETA(epoch): 1994s + [Epoch 168][60/200] loss=0.644962 avg=0.665666 VRAM=38.5GiB | 33.7% done | ETA(epoch): 1860s + [Epoch 168][70/200] loss=0.663806 avg=0.664883 VRAM=38.6GiB | 33.7% done | ETA(epoch): 1726s + [Epoch 168][80/200] loss=0.660006 avg=0.664201 VRAM=38.5GiB | 33.7% done | ETA(epoch): 1593s + [Epoch 168][90/200] loss=0.705888 avg=0.661652 VRAM=38.6GiB | 33.7% done | ETA(epoch): 1460s + [Epoch 168][100/200] loss=0.546785 avg=0.659588 VRAM=38.5GiB | 33.7% done | ETA(epoch): 1327s + [Epoch 168][110/200] loss=0.645607 avg=0.659979 VRAM=38.6GiB | 33.7% done | ETA(epoch): 1194s + [Epoch 168][120/200] loss=0.712562 avg=0.659074 VRAM=38.5GiB | 33.7% done | ETA(epoch): 1061s + [Epoch 168][130/200] loss=0.612880 avg=0.659800 VRAM=38.6GiB | 33.7% done | ETA(epoch): 928s + [Epoch 168][140/200] loss=0.777662 avg=0.663724 VRAM=38.5GiB | 33.7% done | ETA(epoch): 795s + [Epoch 168][150/200] loss=0.647427 avg=0.662710 VRAM=38.6GiB | 33.8% done | ETA(epoch): 663s + [Epoch 168][160/200] loss=0.844778 avg=0.662340 VRAM=38.5GiB | 33.8% done | ETA(epoch): 530s + [Epoch 168][170/200] loss=0.677352 avg=0.661988 VRAM=38.6GiB | 33.8% done | ETA(epoch): 398s + [Epoch 168][180/200] loss=0.614791 avg=0.659488 VRAM=38.5GiB | 33.8% done | ETA(epoch): 265s + [Epoch 168][190/200] loss=0.526154 avg=0.657651 VRAM=38.6GiB | 33.8% done | ETA(epoch): 133s + [Epoch 168][200/200] loss=0.587188 avg=0.659507 VRAM=38.5GiB | 33.8% done | ETA(epoch): 0s + Train loss: 0.659507 (2650.8s) ETA: 16308min + Val loss: 0.701035 [t_0.0-0.2=0.9392 t_0.2-0.4=0.7587 t_0.4-0.6=0.6662 t_0.6-0.8=0.5405 t_0.8-1.0=0.5683] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 169/499 (34% done) --- + [Epoch 169][10/200] loss=0.655099 avg=0.643127 VRAM=38.6GiB | 33.8% done | ETA(epoch): 2515s + [Epoch 169][20/200] loss=0.653157 avg=0.645069 VRAM=38.5GiB | 33.8% done | ETA(epoch): 2383s + [Epoch 169][30/200] loss=0.613976 avg=0.654738 VRAM=38.6GiB | 33.8% done | ETA(epoch): 2250s + [Epoch 169][40/200] loss=0.559667 avg=0.657942 VRAM=38.5GiB | 33.8% done | ETA(epoch): 2118s + [Epoch 169][50/200] loss=0.631944 avg=0.655373 VRAM=38.6GiB | 33.9% done | ETA(epoch): 1985s + [Epoch 169][60/200] loss=0.571684 avg=0.653827 VRAM=38.5GiB | 33.9% done | ETA(epoch): 1853s + [Epoch 169][70/200] loss=0.594005 avg=0.653708 VRAM=38.6GiB | 33.9% done | ETA(epoch): 1721s + [Epoch 169][80/200] loss=0.749580 avg=0.655943 VRAM=38.5GiB | 33.9% done | ETA(epoch): 1589s + [Epoch 169][90/200] loss=0.730926 avg=0.655451 VRAM=38.6GiB | 33.9% done | ETA(epoch): 1459s + [Epoch 169][100/200] loss=0.744829 avg=0.658063 VRAM=38.5GiB | 33.9% done | ETA(epoch): 1326s + [Epoch 169][110/200] loss=0.790430 avg=0.665013 VRAM=38.6GiB | 33.9% done | ETA(epoch): 1194s + [Epoch 169][120/200] loss=0.704457 avg=0.669102 VRAM=38.5GiB | 33.9% done | ETA(epoch): 1061s + [Epoch 169][130/200] loss=0.763868 avg=0.668101 VRAM=38.6GiB | 33.9% done | ETA(epoch): 928s + [Epoch 169][140/200] loss=0.571730 avg=0.667597 VRAM=38.5GiB | 33.9% done | ETA(epoch): 795s + [Epoch 169][150/200] loss=0.617048 avg=0.665982 VRAM=38.6GiB | 34.0% done | ETA(epoch): 663s + [Epoch 169][160/200] loss=0.630825 avg=0.664610 VRAM=38.5GiB | 34.0% done | ETA(epoch): 530s + [Epoch 169][170/200] loss=0.640312 avg=0.663176 VRAM=38.6GiB | 34.0% done | ETA(epoch): 398s + [Epoch 169][180/200] loss=0.608621 avg=0.662890 VRAM=38.5GiB | 34.0% done | ETA(epoch): 265s + [Epoch 169][190/200] loss=0.597254 avg=0.662930 VRAM=38.6GiB | 34.0% done | ETA(epoch): 133s + [Epoch 169][200/200] loss=0.691635 avg=0.663050 VRAM=38.5GiB | 34.0% done | ETA(epoch): 0s + Train loss: 0.663050 (2652.3s) ETA: 16257min + Val loss: 0.691552 [t_0.0-0.2=0.9156 t_0.2-0.4=0.7585 t_0.4-0.6=0.6383 t_0.6-0.8=0.5779 t_0.8-1.0=0.5667] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 170/499 (34% done) --- + [Epoch 170][10/200] loss=0.623981 avg=0.684830 VRAM=38.6GiB | 34.0% done | ETA(epoch): 2511s + [Epoch 170][20/200] loss=0.678825 avg=0.663585 VRAM=38.5GiB | 34.0% done | ETA(epoch): 2384s + [Epoch 170][30/200] loss=0.641888 avg=0.669147 VRAM=38.6GiB | 34.0% done | ETA(epoch): 2251s + [Epoch 170][40/200] loss=0.750847 avg=0.658731 VRAM=38.5GiB | 34.0% done | ETA(epoch): 2120s + [Epoch 170][50/200] loss=0.757562 avg=0.655262 VRAM=38.6GiB | 34.1% done | ETA(epoch): 1987s + [Epoch 170][60/200] loss=0.719551 avg=0.656985 VRAM=38.5GiB | 34.1% done | ETA(epoch): 1854s + [Epoch 170][70/200] loss=0.723635 avg=0.659664 VRAM=38.6GiB | 34.1% done | ETA(epoch): 1721s + [Epoch 170][80/200] loss=0.784065 avg=0.660113 VRAM=38.5GiB | 34.1% done | ETA(epoch): 1589s + [Epoch 170][90/200] loss=0.773708 avg=0.661321 VRAM=38.6GiB | 34.1% done | ETA(epoch): 1457s + [Epoch 170][100/200] loss=0.648716 avg=0.662199 VRAM=38.5GiB | 34.1% done | ETA(epoch): 1324s + [Epoch 170][110/200] loss=0.650307 avg=0.659329 VRAM=38.6GiB | 34.1% done | ETA(epoch): 1192s + [Epoch 170][120/200] loss=0.657718 avg=0.653950 VRAM=38.5GiB | 34.1% done | ETA(epoch): 1059s + [Epoch 170][130/200] loss=0.641764 avg=0.653909 VRAM=38.6GiB | 34.1% done | ETA(epoch): 927s + [Epoch 170][140/200] loss=0.730119 avg=0.652376 VRAM=38.5GiB | 34.1% done | ETA(epoch): 795s + [Epoch 170][150/200] loss=0.741112 avg=0.656103 VRAM=38.6GiB | 34.2% done | ETA(epoch): 662s + [Epoch 170][160/200] loss=0.814285 avg=0.657264 VRAM=38.5GiB | 34.2% done | ETA(epoch): 530s + [Epoch 170][170/200] loss=0.680359 avg=0.658279 VRAM=38.6GiB | 34.2% done | ETA(epoch): 397s + [Epoch 170][180/200] loss=0.630454 avg=0.658487 VRAM=38.5GiB | 34.2% done | ETA(epoch): 265s + [Epoch 170][190/200] loss=0.794697 avg=0.661189 VRAM=38.6GiB | 34.2% done | ETA(epoch): 132s + [Epoch 170][200/200] loss=0.631797 avg=0.659330 VRAM=38.5GiB | 34.2% done | ETA(epoch): 0s + Train loss: 0.659330 (2648.6s) ETA: 16206min + Val loss: 0.703833 [t_0.0-0.2=0.9343 t_0.2-0.4=0.7619 t_0.4-0.6=0.6347 t_0.6-0.8=0.5727 t_0.8-1.0=0.5668] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 171/499 (34% done) --- + [Epoch 171][10/200] loss=0.554094 avg=0.650015 VRAM=38.6GiB | 34.2% done | ETA(epoch): 2517s + [Epoch 171][20/200] loss=0.613476 avg=0.663965 VRAM=38.5GiB | 34.2% done | ETA(epoch): 2385s + [Epoch 171][30/200] loss=0.821945 avg=0.664718 VRAM=38.6GiB | 34.2% done | ETA(epoch): 2251s + [Epoch 171][40/200] loss=0.555263 avg=0.653734 VRAM=38.5GiB | 34.2% done | ETA(epoch): 2119s + [Epoch 171][50/200] loss=0.800891 avg=0.646383 VRAM=38.6GiB | 34.2% done | ETA(epoch): 1986s + [Epoch 171][60/200] loss=0.649865 avg=0.641776 VRAM=38.5GiB | 34.3% done | ETA(epoch): 1854s + [Epoch 171][70/200] loss=0.584048 avg=0.642317 VRAM=38.6GiB | 34.3% done | ETA(epoch): 1722s + [Epoch 171][80/200] loss=0.555200 avg=0.643862 VRAM=38.5GiB | 34.3% done | ETA(epoch): 1589s + [Epoch 171][90/200] loss=0.654432 avg=0.649599 VRAM=38.6GiB | 34.3% done | ETA(epoch): 1457s + [Epoch 171][100/200] loss=0.715955 avg=0.653645 VRAM=38.5GiB | 34.3% done | ETA(epoch): 1324s + [Epoch 171][110/200] loss=0.661564 avg=0.655985 VRAM=38.6GiB | 34.3% done | ETA(epoch): 1192s + [Epoch 171][120/200] loss=0.661409 avg=0.653448 VRAM=38.5GiB | 34.3% done | ETA(epoch): 1059s + [Epoch 171][130/200] loss=0.668935 avg=0.656054 VRAM=38.6GiB | 34.3% done | ETA(epoch): 927s + [Epoch 171][140/200] loss=0.599774 avg=0.653742 VRAM=38.5GiB | 34.3% done | ETA(epoch): 794s + [Epoch 171][150/200] loss=0.706882 avg=0.652549 VRAM=38.6GiB | 34.4% done | ETA(epoch): 662s + [Epoch 171][160/200] loss=0.680093 avg=0.653824 VRAM=38.5GiB | 34.4% done | ETA(epoch): 530s + [Epoch 171][170/200] loss=0.555684 avg=0.653212 VRAM=38.6GiB | 34.4% done | ETA(epoch): 397s + [Epoch 171][180/200] loss=0.731559 avg=0.653902 VRAM=38.5GiB | 34.4% done | ETA(epoch): 265s + [Epoch 171][190/200] loss=0.552960 avg=0.657105 VRAM=38.6GiB | 34.4% done | ETA(epoch): 132s + [Epoch 171][200/200] loss=0.640901 avg=0.657233 VRAM=38.5GiB | 34.4% done | ETA(epoch): 0s + Train loss: 0.657233 (2647.9s) ETA: 16155min + Val loss: 0.700408 [t_0.0-0.2=0.9319 t_0.2-0.4=0.7658 t_0.4-0.6=0.6312 t_0.6-0.8=0.5566 t_0.8-1.0=0.5695] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 172/499 (34% done) --- + [Epoch 172][10/200] loss=0.667590 avg=0.624627 VRAM=38.6GiB | 34.4% done | ETA(epoch): 2513s + [Epoch 172][20/200] loss=0.608096 avg=0.628049 VRAM=38.5GiB | 34.4% done | ETA(epoch): 2384s + [Epoch 172][30/200] loss=0.625555 avg=0.628495 VRAM=38.6GiB | 34.4% done | ETA(epoch): 2251s + [Epoch 172][40/200] loss=0.700648 avg=0.643507 VRAM=38.5GiB | 34.4% done | ETA(epoch): 2119s + [Epoch 172][50/200] loss=0.736706 avg=0.650963 VRAM=38.6GiB | 34.4% done | ETA(epoch): 2002s + [Epoch 172][60/200] loss=0.895704 avg=0.657228 VRAM=38.5GiB | 34.5% done | ETA(epoch): 1866s + [Epoch 172][70/200] loss=0.706144 avg=0.657131 VRAM=38.6GiB | 34.5% done | ETA(epoch): 1731s + [Epoch 172][80/200] loss=0.738768 avg=0.657109 VRAM=38.5GiB | 34.5% done | ETA(epoch): 1597s + [Epoch 172][90/200] loss=0.677185 avg=0.656873 VRAM=38.6GiB | 34.5% done | ETA(epoch): 1463s + [Epoch 172][100/200] loss=0.604552 avg=0.660649 VRAM=38.5GiB | 34.5% done | ETA(epoch): 1329s + [Epoch 172][110/200] loss=0.611664 avg=0.660947 VRAM=38.6GiB | 34.5% done | ETA(epoch): 1196s + [Epoch 172][120/200] loss=0.682482 avg=0.661224 VRAM=38.5GiB | 34.5% done | ETA(epoch): 1063s + [Epoch 172][130/200] loss=0.717650 avg=0.658343 VRAM=38.6GiB | 34.5% done | ETA(epoch): 930s + [Epoch 172][140/200] loss=0.722024 avg=0.656530 VRAM=38.5GiB | 34.5% done | ETA(epoch): 797s + [Epoch 172][150/200] loss=0.568608 avg=0.655429 VRAM=38.6GiB | 34.5% done | ETA(epoch): 664s + [Epoch 172][160/200] loss=0.700886 avg=0.654433 VRAM=38.5GiB | 34.6% done | ETA(epoch): 531s + [Epoch 172][170/200] loss=0.645081 avg=0.654511 VRAM=38.6GiB | 34.6% done | ETA(epoch): 398s + [Epoch 172][180/200] loss=0.596193 avg=0.654659 VRAM=38.5GiB | 34.6% done | ETA(epoch): 265s + [Epoch 172][190/200] loss=0.535673 avg=0.654256 VRAM=38.6GiB | 34.6% done | ETA(epoch): 133s + [Epoch 172][200/200] loss=0.848586 avg=0.656052 VRAM=38.5GiB | 34.6% done | ETA(epoch): 0s + Train loss: 0.656052 (2653.1s) ETA: 16104min + Val loss: 0.698508 [t_0.0-0.2=0.9280 t_0.2-0.4=0.7448 t_0.4-0.6=0.6416 t_0.6-0.8=0.5566 t_0.8-1.0=0.5698] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 173/499 (35% done) --- + [Epoch 173][10/200] loss=0.725467 avg=0.683037 VRAM=38.6GiB | 34.6% done | ETA(epoch): 2557s + [Epoch 173][20/200] loss=0.667367 avg=0.665211 VRAM=38.5GiB | 34.6% done | ETA(epoch): 2402s + [Epoch 173][30/200] loss=0.632117 avg=0.673058 VRAM=38.6GiB | 34.6% done | ETA(epoch): 2263s + [Epoch 173][40/200] loss=0.735078 avg=0.671192 VRAM=38.5GiB | 34.6% done | ETA(epoch): 2126s + [Epoch 173][50/200] loss=0.774526 avg=0.673868 VRAM=38.6GiB | 34.6% done | ETA(epoch): 1992s + [Epoch 173][60/200] loss=0.820584 avg=0.675024 VRAM=38.5GiB | 34.7% done | ETA(epoch): 1858s + [Epoch 173][70/200] loss=0.667854 avg=0.673205 VRAM=38.6GiB | 34.7% done | ETA(epoch): 1724s + [Epoch 173][80/200] loss=0.580962 avg=0.669737 VRAM=38.5GiB | 34.7% done | ETA(epoch): 1592s + [Epoch 173][90/200] loss=0.675264 avg=0.665376 VRAM=38.6GiB | 34.7% done | ETA(epoch): 1459s + [Epoch 173][100/200] loss=0.568625 avg=0.663028 VRAM=38.5GiB | 34.7% done | ETA(epoch): 1331s + [Epoch 173][110/200] loss=0.657085 avg=0.660868 VRAM=38.6GiB | 34.7% done | ETA(epoch): 1197s + [Epoch 173][120/200] loss=0.869768 avg=0.664921 VRAM=38.5GiB | 34.7% done | ETA(epoch): 1064s + [Epoch 173][130/200] loss=0.706408 avg=0.666993 VRAM=38.6GiB | 34.7% done | ETA(epoch): 931s + [Epoch 173][140/200] loss=0.710066 avg=0.669228 VRAM=38.5GiB | 34.7% done | ETA(epoch): 797s + [Epoch 173][150/200] loss=0.618761 avg=0.666679 VRAM=38.6GiB | 34.8% done | ETA(epoch): 664s + [Epoch 173][160/200] loss=0.782213 avg=0.667811 VRAM=38.5GiB | 34.8% done | ETA(epoch): 531s + [Epoch 173][170/200] loss=0.566654 avg=0.665992 VRAM=38.6GiB | 34.8% done | ETA(epoch): 398s + [Epoch 173][180/200] loss=0.651124 avg=0.664626 VRAM=38.5GiB | 34.8% done | ETA(epoch): 266s + [Epoch 173][190/200] loss=0.641058 avg=0.663911 VRAM=38.6GiB | 34.8% done | ETA(epoch): 133s + [Epoch 173][200/200] loss=0.729645 avg=0.663923 VRAM=38.5GiB | 34.8% done | ETA(epoch): 0s + Train loss: 0.663923 (2655.3s) ETA: 16053min + Val loss: 0.680973 [t_0.0-0.2=0.9354 t_0.2-0.4=0.7431 t_0.4-0.6=0.6478 t_0.6-0.8=0.5567 t_0.8-1.0=0.5608] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 25.0/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 174/499 (35% done) --- + [Epoch 174][10/200] loss=0.801293 avg=0.676790 VRAM=38.6GiB | 34.8% done | ETA(epoch): 2515s + [Epoch 174][20/200] loss=0.578158 avg=0.661958 VRAM=38.5GiB | 34.8% done | ETA(epoch): 2383s + [Epoch 174][30/200] loss=0.684807 avg=0.653934 VRAM=38.6GiB | 34.8% done | ETA(epoch): 2252s + [Epoch 174][40/200] loss=0.683450 avg=0.653464 VRAM=38.5GiB | 34.8% done | ETA(epoch): 2119s + [Epoch 174][50/200] loss=0.579607 avg=0.651979 VRAM=38.6GiB | 34.8% done | ETA(epoch): 1987s + [Epoch 174][60/200] loss=0.755565 avg=0.657386 VRAM=38.5GiB | 34.9% done | ETA(epoch): 1861s + [Epoch 174][70/200] loss=0.520564 avg=0.654454 VRAM=38.6GiB | 34.9% done | ETA(epoch): 1727s + [Epoch 174][80/200] loss=0.624148 avg=0.654996 VRAM=38.5GiB | 34.9% done | ETA(epoch): 1593s + [Epoch 174][90/200] loss=0.527620 avg=0.653000 VRAM=38.6GiB | 34.9% done | ETA(epoch): 1460s + [Epoch 174][100/200] loss=0.641369 avg=0.651054 VRAM=38.5GiB | 34.9% done | ETA(epoch): 1327s + [Epoch 174][110/200] loss=0.688002 avg=0.652339 VRAM=38.6GiB | 34.9% done | ETA(epoch): 1194s + [Epoch 174][120/200] loss=0.646186 avg=0.653730 VRAM=38.5GiB | 34.9% done | ETA(epoch): 1061s + [Epoch 174][130/200] loss=0.662780 avg=0.652243 VRAM=38.6GiB | 34.9% done | ETA(epoch): 929s + [Epoch 174][140/200] loss=0.737257 avg=0.655474 VRAM=38.5GiB | 34.9% done | ETA(epoch): 796s + [Epoch 174][150/200] loss=0.685773 avg=0.655011 VRAM=38.6GiB | 34.9% done | ETA(epoch): 663s + [Epoch 174][160/200] loss=0.613989 avg=0.653908 VRAM=38.5GiB | 35.0% done | ETA(epoch): 530s + [Epoch 174][170/200] loss=0.726099 avg=0.654937 VRAM=38.6GiB | 35.0% done | ETA(epoch): 398s + [Epoch 174][180/200] loss=0.702963 avg=0.656079 VRAM=38.5GiB | 35.0% done | ETA(epoch): 265s + [Epoch 174][190/200] loss=0.786233 avg=0.658744 VRAM=38.6GiB | 35.0% done | ETA(epoch): 133s + [Epoch 174][200/200] loss=0.573254 avg=0.659171 VRAM=38.5GiB | 35.0% done | ETA(epoch): 0s + Train loss: 0.659171 (2652.0s) ETA: 16002min + Val loss: 0.689159 [t_0.0-0.2=0.9284 t_0.2-0.4=0.7428 t_0.4-0.6=0.6425 t_0.6-0.8=0.5634 t_0.8-1.0=0.5737] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 175/499 (35% done) --- + [Epoch 175][10/200] loss=0.720959 avg=0.666970 VRAM=38.6GiB | 35.0% done | ETA(epoch): 2512s + [Epoch 175][20/200] loss=0.856523 avg=0.662408 VRAM=38.5GiB | 35.0% done | ETA(epoch): 2382s + [Epoch 175][30/200] loss=0.558916 avg=0.653537 VRAM=38.6GiB | 35.0% done | ETA(epoch): 2252s + [Epoch 175][40/200] loss=0.762860 avg=0.662124 VRAM=38.5GiB | 35.0% done | ETA(epoch): 2119s + [Epoch 175][50/200] loss=0.664553 avg=0.662284 VRAM=38.6GiB | 35.0% done | ETA(epoch): 1987s + [Epoch 175][60/200] loss=0.777814 avg=0.665906 VRAM=38.5GiB | 35.1% done | ETA(epoch): 1854s + [Epoch 175][70/200] loss=0.574492 avg=0.664937 VRAM=38.6GiB | 35.1% done | ETA(epoch): 1722s + [Epoch 175][80/200] loss=0.561960 avg=0.661271 VRAM=38.5GiB | 35.1% done | ETA(epoch): 1589s + [Epoch 175][90/200] loss=0.761134 avg=0.663290 VRAM=38.6GiB | 35.1% done | ETA(epoch): 1457s + [Epoch 175][100/200] loss=0.679342 avg=0.663041 VRAM=38.5GiB | 35.1% done | ETA(epoch): 1324s + [Epoch 175][110/200] loss=0.571970 avg=0.660496 VRAM=38.6GiB | 35.1% done | ETA(epoch): 1196s + [Epoch 175][120/200] loss=0.579296 avg=0.655660 VRAM=38.5GiB | 35.1% done | ETA(epoch): 1063s + [Epoch 175][130/200] loss=0.766254 avg=0.657025 VRAM=38.6GiB | 35.1% done | ETA(epoch): 930s + [Epoch 175][140/200] loss=0.578879 avg=0.654151 VRAM=38.5GiB | 35.1% done | ETA(epoch): 797s + [Epoch 175][150/200] loss=0.721075 avg=0.655735 VRAM=38.6GiB | 35.1% done | ETA(epoch): 664s + [Epoch 175][160/200] loss=0.660692 avg=0.656685 VRAM=38.5GiB | 35.2% done | ETA(epoch): 531s + [Epoch 175][170/200] loss=0.711476 avg=0.656431 VRAM=38.6GiB | 35.2% done | ETA(epoch): 398s + [Epoch 175][180/200] loss=0.705101 avg=0.657531 VRAM=38.5GiB | 35.2% done | ETA(epoch): 265s + [Epoch 175][190/200] loss=0.574594 avg=0.657599 VRAM=38.6GiB | 35.2% done | ETA(epoch): 133s + [Epoch 175][200/200] loss=0.785828 avg=0.660082 VRAM=38.5GiB | 35.2% done | ETA(epoch): 0s + Train loss: 0.660082 (2653.6s) ETA: 15951min + Val loss: 0.664574 [t_0.0-0.2=0.9101 t_0.2-0.4=0.7577 t_0.4-0.6=0.6417 t_0.6-0.8=0.5605 t_0.8-1.0=0.5639] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 24.9/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 176/499 (35% done) --- + [Epoch 176][10/200] loss=0.575494 avg=0.613736 VRAM=38.6GiB | 35.2% done | ETA(epoch): 2523s + [Epoch 176][20/200] loss=0.860809 avg=0.649329 VRAM=38.5GiB | 35.2% done | ETA(epoch): 2386s + [Epoch 176][30/200] loss=0.646366 avg=0.650091 VRAM=38.6GiB | 35.2% done | ETA(epoch): 2254s + [Epoch 176][40/200] loss=0.675809 avg=0.643231 VRAM=38.5GiB | 35.2% done | ETA(epoch): 2120s + [Epoch 176][50/200] loss=0.672154 avg=0.639453 VRAM=38.6GiB | 35.2% done | ETA(epoch): 1987s + [Epoch 176][60/200] loss=0.764512 avg=0.645632 VRAM=38.5GiB | 35.3% done | ETA(epoch): 1855s + [Epoch 176][70/200] loss=0.586700 avg=0.648064 VRAM=38.6GiB | 35.3% done | ETA(epoch): 1722s + [Epoch 176][80/200] loss=0.672482 avg=0.653021 VRAM=38.5GiB | 35.3% done | ETA(epoch): 1593s + [Epoch 176][90/200] loss=0.673941 avg=0.657541 VRAM=38.6GiB | 35.3% done | ETA(epoch): 1460s + [Epoch 176][100/200] loss=0.655135 avg=0.658407 VRAM=38.5GiB | 35.3% done | ETA(epoch): 1327s + [Epoch 176][110/200] loss=0.585655 avg=0.656351 VRAM=38.6GiB | 35.3% done | ETA(epoch): 1194s + [Epoch 176][120/200] loss=0.650624 avg=0.656314 VRAM=38.5GiB | 35.3% done | ETA(epoch): 1061s + [Epoch 176][130/200] loss=0.578166 avg=0.656465 VRAM=38.6GiB | 35.3% done | ETA(epoch): 929s + [Epoch 176][140/200] loss=0.580232 avg=0.655301 VRAM=38.5GiB | 35.3% done | ETA(epoch): 796s + [Epoch 176][150/200] loss=0.566431 avg=0.654276 VRAM=38.6GiB | 35.4% done | ETA(epoch): 663s + [Epoch 176][160/200] loss=0.809267 avg=0.653481 VRAM=38.5GiB | 35.4% done | ETA(epoch): 530s + [Epoch 176][170/200] loss=0.675041 avg=0.652687 VRAM=38.6GiB | 35.4% done | ETA(epoch): 398s + [Epoch 176][180/200] loss=0.677134 avg=0.652078 VRAM=38.5GiB | 35.4% done | ETA(epoch): 265s + [Epoch 176][190/200] loss=0.631648 avg=0.652337 VRAM=38.6GiB | 35.4% done | ETA(epoch): 133s + [Epoch 176][200/200] loss=0.714423 avg=0.653949 VRAM=38.5GiB | 35.4% done | ETA(epoch): 0s + Train loss: 0.653949 (2653.8s) ETA: 15901min + Val loss: 0.705895 [t_0.0-0.2=0.9380 t_0.2-0.4=0.7591 t_0.4-0.6=0.6530 t_0.6-0.8=0.5567 t_0.8-1.0=0.5553] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 25.0/188.5 GiB (13.2%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 177/499 (35% done) --- + [Epoch 177][10/200] loss=0.631914 avg=0.640520 VRAM=38.6GiB | 35.4% done | ETA(epoch): 2516s + [Epoch 177][20/200] loss=0.546943 avg=0.626243 VRAM=38.5GiB | 35.4% done | ETA(epoch): 2383s + [Epoch 177][30/200] loss=0.629298 avg=0.635777 VRAM=38.6GiB | 35.4% done | ETA(epoch): 2251s + [Epoch 177][40/200] loss=0.663416 avg=0.643053 VRAM=38.5GiB | 35.4% done | ETA(epoch): 2138s + [Epoch 177][50/200] loss=0.742212 avg=0.643293 VRAM=38.6GiB | 35.4% done | ETA(epoch): 2001s + [Epoch 177][60/200] loss=0.603017 avg=0.647742 VRAM=38.5GiB | 35.5% done | ETA(epoch): 1866s + [Epoch 177][70/200] loss=0.617683 avg=0.646948 VRAM=38.6GiB | 35.5% done | ETA(epoch): 1731s + [Epoch 177][80/200] loss=0.677145 avg=0.649668 VRAM=38.5GiB | 35.5% done | ETA(epoch): 1596s + [Epoch 177][90/200] loss=0.650586 avg=0.652347 VRAM=38.6GiB | 35.5% done | ETA(epoch): 1462s + [Epoch 177][100/200] loss=0.619897 avg=0.651863 VRAM=38.5GiB | 35.5% done | ETA(epoch): 1329s + [Epoch 177][110/200] loss=0.642760 avg=0.653929 VRAM=38.6GiB | 35.5% done | ETA(epoch): 1196s + [Epoch 177][120/200] loss=0.563594 avg=0.653551 VRAM=38.5GiB | 35.5% done | ETA(epoch): 1062s + [Epoch 177][130/200] loss=0.641217 avg=0.655770 VRAM=38.6GiB | 35.5% done | ETA(epoch): 931s + [Epoch 177][140/200] loss=0.646307 avg=0.654853 VRAM=38.5GiB | 35.5% done | ETA(epoch): 798s + [Epoch 177][150/200] loss=0.578115 avg=0.653681 VRAM=38.6GiB | 35.5% done | ETA(epoch): 665s + [Epoch 177][160/200] loss=0.726187 avg=0.656644 VRAM=38.5GiB | 35.6% done | ETA(epoch): 532s + [Epoch 177][170/200] loss=0.678397 avg=0.655125 VRAM=38.6GiB | 35.6% done | ETA(epoch): 399s + [Epoch 177][180/200] loss=0.571963 avg=0.655180 VRAM=38.5GiB | 35.6% done | ETA(epoch): 266s + [Epoch 177][190/200] loss=0.643581 avg=0.654400 VRAM=38.6GiB | 35.6% done | ETA(epoch): 133s + [Epoch 177][200/200] loss=0.636407 avg=0.653170 VRAM=38.5GiB | 35.6% done | ETA(epoch): 0s + Train loss: 0.653170 (2655.4s) ETA: 15850min + Val loss: 0.689681 [t_0.0-0.2=0.9679 t_0.2-0.4=0.7459 t_0.4-0.6=0.6327 t_0.6-0.8=0.5609 t_0.8-1.0=0.5643] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 25.1/188.5 GiB (13.3%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 178/499 (36% done) --- + [Epoch 178][10/200] loss=0.707132 avg=0.652618 VRAM=38.6GiB | 35.6% done | ETA(epoch): 2512s + [Epoch 178][20/200] loss=0.816184 avg=0.664064 VRAM=38.5GiB | 35.6% done | ETA(epoch): 2382s + [Epoch 178][30/200] loss=0.581258 avg=0.637315 VRAM=38.6GiB | 35.6% done | ETA(epoch): 2249s + [Epoch 178][40/200] loss=0.801861 avg=0.636893 VRAM=38.5GiB | 35.6% done | ETA(epoch): 2116s + [Epoch 178][50/200] loss=0.764435 avg=0.636085 VRAM=38.6GiB | 35.6% done | ETA(epoch): 1985s + [Epoch 178][60/200] loss=0.676624 avg=0.639560 VRAM=38.5GiB | 35.7% done | ETA(epoch): 1852s + [Epoch 178][70/200] loss=0.743116 avg=0.640244 VRAM=38.6GiB | 35.7% done | ETA(epoch): 1721s + [Epoch 178][80/200] loss=0.660349 avg=0.644070 VRAM=38.5GiB | 35.7% done | ETA(epoch): 1588s + [Epoch 178][90/200] loss=0.594994 avg=0.651276 VRAM=38.6GiB | 35.7% done | ETA(epoch): 1456s + [Epoch 178][100/200] loss=0.907634 avg=0.654774 VRAM=38.5GiB | 35.7% done | ETA(epoch): 1324s + [Epoch 178][110/200] loss=0.595575 avg=0.656058 VRAM=38.6GiB | 35.7% done | ETA(epoch): 1191s + [Epoch 178][120/200] loss=0.637577 avg=0.653732 VRAM=38.5GiB | 35.7% done | ETA(epoch): 1059s + [Epoch 178][130/200] loss=0.657934 avg=0.656715 VRAM=38.6GiB | 35.7% done | ETA(epoch): 927s + [Epoch 178][140/200] loss=0.836899 avg=0.659179 VRAM=38.5GiB | 35.7% done | ETA(epoch): 794s + [Epoch 178][150/200] loss=0.663621 avg=0.662412 VRAM=38.6GiB | 35.8% done | ETA(epoch): 662s + [Epoch 178][160/200] loss=0.699951 avg=0.662455 VRAM=38.5GiB | 35.8% done | ETA(epoch): 530s + [Epoch 178][170/200] loss=0.536395 avg=0.659885 VRAM=38.6GiB | 35.8% done | ETA(epoch): 397s + [Epoch 178][180/200] loss=0.659350 avg=0.658236 VRAM=38.5GiB | 35.8% done | ETA(epoch): 265s + [Epoch 178][190/200] loss=0.662695 avg=0.657458 VRAM=38.6GiB | 35.8% done | ETA(epoch): 132s + [Epoch 178][200/200] loss=0.631763 avg=0.653929 VRAM=38.5GiB | 35.8% done | ETA(epoch): 0s + Train loss: 0.653929 (2647.6s) ETA: 15799min + Val loss: 0.684166 [t_0.0-0.2=0.9402 t_0.2-0.4=0.7432 t_0.4-0.6=0.6492 t_0.6-0.8=0.5596 t_0.8-1.0=0.5682] + Checkpoint saved: output/lora_rank_128_mlp_1k_h100/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 26.2/188.5 GiB (13.9%) | VRAM: 38.2/93.1 GiB (41.1%) + +--- Epoch 179/499 (36% done) --- + [Epoch 179][10/200] loss=0.623213 avg=0.662594 VRAM=38.6GiB | 35.8% done | ETA(epoch): 2519s + [Epoch 179][20/200] loss=0.575161 avg=0.646123 VRAM=38.5GiB | 35.8% done | ETA(epoch): 2384s + [Epoch 179][30/200] loss=0.763819 avg=0.648122 VRAM=38.6GiB | 35.8% done | ETA(epoch): 2251s + [Epoch 179][40/200] loss=0.669066 avg=0.651008 VRAM=38.5GiB | 35.8% done | ETA(epoch): 2119s diff --git a/test_single_gpu_20260415_162558.log b/test_single_gpu_20260415_162558.log new file mode 100644 index 0000000000000000000000000000000000000000..e780a83540bc4b9c57dec92e221da7f540a3d42e --- /dev/null +++ b/test_single_gpu_20260415_162558.log @@ -0,0 +1,93 @@ +Run dir : output/_smoke_test_1gpu +Log file: output/_smoke_test_1gpu/train.log +GPU: NVIDIA GeForce RTX 5090 | VRAM: 31.4 GiB | PyTorch: 2.11.0+cu130 + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/_smoke_test_1gpu + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 1 + batch_size 4 + learning_rate 0.0003 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 1 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project _smoke_test_1gpu + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.8/188.5 GiB (6.3%) | VRAM: 0.0/31.4 GiB (0.0%) + +============================================================ + [1/8] Text Embeddings +============================================================ + Composed prompt (575 chars): + Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite imagery, natural daylight, buildings: detaile... + +=== Precomputing Global Text Embedding === + Prompt: Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite image... + Loading safetensors: weights/mistral_3_small_flux2_fp8.safetensors + Building tokenizer from embedded tekken_model ... + Detected: 30 layers, hidden=5120, heads=32, kv_heads=8, ffn=32768, vocab=131072 + Initialising MistralModel (30 layers)... + Dequantizing FP8 weights → bf16 ... +Traceback (most recent call last): + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1297, in + main() + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 731, in main + precompute_single_prompt_embeddings( + File "/home/xg_wang_group/SynthUrbanSAT/scripts/text_encoder.py", line 449, in precompute_single_prompt_embeddings + text_encoder, tokenizer = load_text_encoder(model_path, device, dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/text_encoder.py", line 115, in load_text_encoder + return _load_from_safetensors(model_path, device, dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/text_encoder.py", line 213, in _load_from_safetensors + text_encoder = text_encoder.to(device=device, dtype=dtype).eval() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3574, in to + return super().to(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1384, in to + return self._apply(convert) + ^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 934, in _apply + module._apply(fn) + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 934, in _apply + module._apply(fn) + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 934, in _apply + module._apply(fn) + [Previous line repeated 1 more time] + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 965, in _apply + param_applied = fn(param) + ^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1370, in convert + return t.to( + ^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 320.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 208.56 MiB is free. Including non-PyTorch memory, this process has 31.13 GiB memory in use. Of the allocated memory 30.65 GiB is allocated by PyTorch, and 1.45 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf) diff --git a/test_single_gpu_20260415_165338.log b/test_single_gpu_20260415_165338.log new file mode 100644 index 0000000000000000000000000000000000000000..5d937f7e199bf943d731b8cf73785e645387d5a9 --- /dev/null +++ b/test_single_gpu_20260415_165338.log @@ -0,0 +1,58 @@ +Run dir : output/_smoke_test_1gpu +Log file: output/_smoke_test_1gpu/train.log +GPU: NVIDIA GeForce RTX 5090 | VRAM: 31.4 GiB | PyTorch: 2.11.0+cu130 + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/_smoke_test_1gpu + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 1 + batch_size 4 + learning_rate 0.0003 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 1 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project _smoke_test_1gpu + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 10.7/188.5 GiB (5.7%) | VRAM: 0.0/31.4 GiB (0.0%) + +============================================================ + [1/8] Text Embeddings +============================================================ + Composed prompt (575 chars): + Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite imagery, natural daylight, buildings: detaile... + +=== Precomputing Global Text Embedding === + Prompt: Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite image... + Loading safetensors: weights/mistral_3_small_flux2_fp8.safetensors + Building tokenizer from embedded tekken_model ... + Detected: 30 layers, hidden=5120, heads=32, kv_heads=8, ffn=32768, vocab=131072 + Initialising MistralModel (30 layers)... diff --git a/test_single_gpu_20260415_165414.log b/test_single_gpu_20260415_165414.log new file mode 100644 index 0000000000000000000000000000000000000000..86a0e6aa9fb6e4963436ff2a88e2ab221f13c307 --- /dev/null +++ b/test_single_gpu_20260415_165414.log @@ -0,0 +1,93 @@ +Run dir : output/_smoke_test_1gpu +Log file: output/_smoke_test_1gpu/train.log +GPU: NVIDIA GeForce RTX 5090 | VRAM: 31.4 GiB | PyTorch: 2.11.0+cu130 + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/_smoke_test_1gpu + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 1 + batch_size 4 + learning_rate 0.0003 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 1 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project _smoke_test_1gpu + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 0.0/31.4 GiB (0.0%) + +============================================================ + [1/8] Text Embeddings +============================================================ + Composed prompt (575 chars): + Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite imagery, natural daylight, buildings: detaile... + +=== Precomputing Global Text Embedding === + Prompt: Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite image... + Loading safetensors: weights/mistral_3_small_flux2_fp8.safetensors + Building tokenizer from embedded tekken_model ... + Detected: 30 layers, hidden=5120, heads=32, kv_heads=8, ffn=32768, vocab=131072 + Initialising MistralModel (30 layers)... + Dequantizing FP8 weights → bf16 ... +Traceback (most recent call last): + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 1297, in + main() + File "/home/xg_wang_group/SynthUrbanSAT/train_script.py", line 731, in main + precompute_single_prompt_embeddings( + File "/home/xg_wang_group/SynthUrbanSAT/scripts/text_encoder.py", line 449, in precompute_single_prompt_embeddings + text_encoder, tokenizer = load_text_encoder(model_path, device, dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/text_encoder.py", line 115, in load_text_encoder + return _load_from_safetensors(model_path, device, dtype) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/SynthUrbanSAT/scripts/text_encoder.py", line 213, in _load_from_safetensors + text_encoder = text_encoder.to(device=device, dtype=dtype).eval() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3574, in to + return super().to(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1384, in to + return self._apply(convert) + ^^^^^^^^^^^^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 934, in _apply + module._apply(fn) + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 934, in _apply + module._apply(fn) + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 934, in _apply + module._apply(fn) + [Previous line repeated 1 more time] + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 965, in _apply + param_applied = fn(param) + ^^^^^^^^^ + File "/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1370, in convert + return t.to( + ^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 320.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 208.56 MiB is free. Including non-PyTorch memory, this process has 31.13 GiB memory in use. Of the allocated memory 30.65 GiB is allocated by PyTorch, and 1.45 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf) diff --git a/test_single_gpu_20260415_165844.log b/test_single_gpu_20260415_165844.log new file mode 100644 index 0000000000000000000000000000000000000000..e3d67700558813f32be0b3e03a7528147627abfd --- /dev/null +++ b/test_single_gpu_20260415_165844.log @@ -0,0 +1,175 @@ +Run dir : output/_smoke_test_1gpu +Log file: output/_smoke_test_1gpu/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/_smoke_test_1gpu + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 1 + batch_size 4 + learning_rate 0.0003 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 1 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity + wandb_project _smoke_test_1gpu + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 0.0/93.1 GiB (0.0%) + +============================================================ + [1/8] Text Embeddings +============================================================ + Composed prompt (575 chars): + Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite imagery, natural daylight, buildings: detaile... + +=== Precomputing Global Text Embedding === + Prompt: Aerial top-down satellite view of American urban area, Google Earth style, 8k resolution, photorealistic satellite image... + Loading safetensors: weights/mistral_3_small_flux2_fp8.safetensors + Building tokenizer from embedded tekken_model ... + Detected: 30 layers, hidden=5120, heads=32, kv_heads=8, ffn=32768, vocab=131072 + Initialising MistralModel (30 layers)... + Dequantizing FP8 weights → bf16 ... + Text encoder: 17.3B params, VRAM: 34.69 GB + Embedding shape: torch.Size([512, 15360]) + Saved global embedding → output/text_embeddings_global.pt + Text encoder unloaded. VRAM: 34.7 GB +=== Global Text Embedding Done === + + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.1s), VRAM: 0.19 GiB +[MEM @ after VAE] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (59.7s), VRAM: 37.90 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 17.8/188.5 GiB (9.5%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ +HDC²A: 52.4M params +Control: 0.0M params +Total trainable: 52.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=32, alpha=32.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 9.83M + +Parameter Statistics: + HDC²A Adapter: total=52.4M trainable=52.4M + ControlNet (frozen): total=4143.4M LoRA trainable=9.83M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 52.4M + LoRA 9.83M = 62.19M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=3.00e-04, backbone_lr=0.00e+00 + param_group 'adapter': 112 tensors, lr=3.00e-04 + Scheduler: 400 warmup steps → cosine over ~25 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0481, std=0.5195 + VRAM peak (forward): 68.47 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.376319 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 49.21 GiB + [test 4/4] Gradient flow check... + HDC²A: 112/112 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.002655 + depth_encoder.conv_stem.6.weight: 0.002594 + W_d.weight: 0.002533 + W_s.weight: 0.002380 + fusion_blocks.0.ffn_dep.2.weight: 0.002014 + Test result: PASSED +[MEM @ after test] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.1/93.1 GiB (40.9%) + +*** --test passed: all models loaded, forward test OK. Exiting. *** diff --git a/training log/trainlog(lora128dim512) b/training log/trainlog(lora128dim512) new file mode 100644 index 0000000000000000000000000000000000000000..cfc5469abe14123e51ac878b28c15cb28a5c7e9f --- /dev/null +++ b/training log/trainlog(lora128dim512) @@ -0,0 +1,4400 @@ + +====================================================================== +Command: train_script.py --name lora_128_dim_512_H100 --batch-size 6 --adapter-lr 1e-3 --num-epochs 300 --lora-lr 1e-4 --adapter-mlp --lora-rank 128 --fusion-dim 512 --seed 42 +====================================================================== +Run dir : output/lora_128_dim_512_H100 +Log file: output/lora_128_dim_512_H100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.3s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (0.8s) + m wandb: setting up run eo9bzlri (1.3s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260428_190804-eo9bzlri +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_128_dim_512_H100-train-20260428-190804 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100/runs/eo9bzlri + +Run Summary: + name lora_128_dim_512_H100 + fusion MLP (lightweight) + LoRA ON (rank=128) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_128_dim_512_H100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 512 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 300 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_128_dim_512_H100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 12.7/188.5 GiB (6.7%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.7s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 13.1/188.5 GiB (7.0%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (59.0s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 14.7/188.5 GiB (7.8%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 9.4M params +Control: 0.0M params +Total trainable: 9.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=128, alpha=128.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 39.32M + +Parameter Statistics: + HDC²A Adapter: total=9.4M trainable=9.4M + ControlNet (frozen): total=4172.8M LoRA trainable=39.32M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 9.4M + LoRA 39.32M = 48.77M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~4800 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0347, std=0.5039 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.350869 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.71 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.035889 + depth_encoder.conv_stem.6.weight: 0.032471 + semantic_encoder.conv_stem.3.weight: 0.022705 + depth_encoder.conv_stem.3.weight: 0.020752 + W_s.weight: 0.019043 + Test result: PASSED +[MEM @ after test] RAM: 15.2/188.5 GiB (8.1%) | VRAM: 38.0/93.1 GiB (40.8%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 672.097 GFLOPs transformer: 273441.526 GFLOPs total: 274113.623 GFLOPs + per-sample forward FLOPs: 274113.623 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 300 epochs × 66 steps = 19800 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 1980, 3960, 5940, 7920, 9900, 11879, 13859, 15839, 17819, 19799] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/299 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/66] loss=1.009864 avg=0.895808 VRAM=38.7GiB | 0.1% done | ETA(epoch): 6744s + [Epoch 0][20/66] loss=0.972565 avg=0.910056 VRAM=38.6GiB | 0.1% done | ETA(epoch): 3213s + [Epoch 0][30/66] loss=0.984125 avg=0.915839 VRAM=38.7GiB | 0.2% done | ETA(epoch): 1907s + [Epoch 0][40/66] loss=1.036940 avg=0.920781 VRAM=38.6GiB | 0.2% done | ETA(epoch): 1158s + [Epoch 0][50/66] loss=1.007875 avg=0.923174 VRAM=38.7GiB | 0.3% done | ETA(epoch): 632s + [Epoch 0][60/66] loss=0.974190 avg=0.914450 VRAM=38.6GiB | 0.3% done | ETA(epoch): 217s + Train loss: 0.913676 (2286.2s) ETA: 11415min + Val loss: 0.983590 [t_0.0-0.2=1.0969 t_0.2-0.4=1.0573 t_0.4-0.6=0.8935 t_0.6-0.8=0.7610 t_0.8-1.0=1.2208] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 1/299 (0% done) --- + [Epoch 1][10/66] loss=0.869710 avg=0.910784 VRAM=38.7GiB | 0.4% done | ETA(epoch): 1084s + [Epoch 1][20/66] loss=0.939176 avg=0.905765 VRAM=38.6GiB | 0.4% done | ETA(epoch): 887s + [Epoch 1][30/66] loss=0.977605 avg=0.907435 VRAM=38.7GiB | 0.5% done | ETA(epoch): 694s + [Epoch 1][40/66] loss=0.936588 avg=0.905888 VRAM=38.6GiB | 0.5% done | ETA(epoch): 501s + [Epoch 1][50/66] loss=0.912928 avg=0.907713 VRAM=38.7GiB | 0.6% done | ETA(epoch): 308s + [Epoch 1][60/66] loss=0.872662 avg=0.908097 VRAM=38.6GiB | 0.6% done | ETA(epoch): 116s + Train loss: 0.906855 (1271.3s) ETA: 9079min + Val loss: 0.971690 [t_0.0-0.2=1.0884 t_0.2-0.4=1.0387 t_0.4-0.6=0.8795 t_0.6-0.8=0.7632 t_0.8-1.0=1.0611] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 2/299 (1% done) --- + [Epoch 2][10/66] loss=0.944325 avg=0.930917 VRAM=38.7GiB | 0.7% done | ETA(epoch): 1078s + [Epoch 2][20/66] loss=0.795080 avg=0.934827 VRAM=38.6GiB | 0.8% done | ETA(epoch): 885s + [Epoch 2][30/66] loss=0.938770 avg=0.932447 VRAM=38.7GiB | 0.8% done | ETA(epoch): 693s + [Epoch 2][40/66] loss=0.865406 avg=0.926226 VRAM=38.6GiB | 0.9% done | ETA(epoch): 500s + [Epoch 2][50/66] loss=0.964584 avg=0.929361 VRAM=38.7GiB | 0.9% done | ETA(epoch): 309s + [Epoch 2][60/66] loss=0.915467 avg=0.924398 VRAM=38.6GiB | 1.0% done | ETA(epoch): 116s + Train loss: 0.921698 (1275.3s) ETA: 8293min + Val loss: 0.955505 [t_0.0-0.2=1.0840 t_0.2-0.4=1.0176 t_0.4-0.6=0.9122 t_0.6-0.8=0.7626 t_0.8-1.0=0.9271] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0002 (BEST) +[MEM @ epoch 2 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 3/299 (1% done) --- + [Epoch 3][10/66] loss=0.975400 avg=0.917040 VRAM=38.7GiB | 1.1% done | ETA(epoch): 1077s + [Epoch 3][20/66] loss=0.810875 avg=0.903594 VRAM=38.6GiB | 1.1% done | ETA(epoch): 886s + [Epoch 3][30/66] loss=0.903459 avg=0.898993 VRAM=38.7GiB | 1.2% done | ETA(epoch): 693s + [Epoch 3][40/66] loss=0.933859 avg=0.905058 VRAM=38.6GiB | 1.2% done | ETA(epoch): 501s + [Epoch 3][50/66] loss=0.958282 avg=0.903798 VRAM=38.7GiB | 1.3% done | ETA(epoch): 308s + [Epoch 3][60/66] loss=0.871687 avg=0.903271 VRAM=38.6GiB | 1.3% done | ETA(epoch): 116s + Train loss: 0.903173 (1271.0s) ETA: 7883min + Val loss: 0.983942 [t_0.0-0.2=1.0964 t_0.2-0.4=1.0031 t_0.4-0.6=0.8597 t_0.6-0.8=0.7730 t_0.8-1.0=1.0041] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 4/299 (1% done) --- + [Epoch 4][10/66] loss=0.892058 avg=0.880796 VRAM=38.7GiB | 1.4% done | ETA(epoch): 1077s + [Epoch 4][20/66] loss=0.866788 avg=0.890880 VRAM=38.6GiB | 1.4% done | ETA(epoch): 885s + [Epoch 4][30/66] loss=0.859495 avg=0.887039 VRAM=38.7GiB | 1.5% done | ETA(epoch): 693s + [Epoch 4][40/66] loss=0.811758 avg=0.882804 VRAM=38.6GiB | 1.5% done | ETA(epoch): 501s + [Epoch 4][50/66] loss=0.952058 avg=0.887590 VRAM=38.7GiB | 1.6% done | ETA(epoch): 308s + [Epoch 4][60/66] loss=0.938833 avg=0.887905 VRAM=38.6GiB | 1.6% done | ETA(epoch): 116s + Train loss: 0.891030 (1270.7s) ETA: 7628min + Val loss: 0.901731 [t_0.0-0.2=1.0986 t_0.2-0.4=1.0158 t_0.4-0.6=0.7993 t_0.6-0.8=0.7423 t_0.8-1.0=0.9023] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0004 (BEST) + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 5/299 (2% done) --- + [Epoch 5][10/66] loss=0.936078 avg=0.867243 VRAM=38.7GiB | 1.7% done | ETA(epoch): 1077s + [Epoch 5][20/66] loss=0.836313 avg=0.883254 VRAM=38.6GiB | 1.8% done | ETA(epoch): 885s + [Epoch 5][30/66] loss=0.906815 avg=0.886041 VRAM=38.7GiB | 1.8% done | ETA(epoch): 695s + [Epoch 5][40/66] loss=0.975250 avg=0.891003 VRAM=38.6GiB | 1.9% done | ETA(epoch): 502s + [Epoch 5][50/66] loss=0.932049 avg=0.889707 VRAM=38.7GiB | 1.9% done | ETA(epoch): 309s + [Epoch 5][60/66] loss=0.933814 avg=0.895149 VRAM=38.6GiB | 2.0% done | ETA(epoch): 116s + Train loss: 0.893848 (1273.0s) ETA: 7453min + Val loss: 0.893321 [t_0.0-0.2=1.0984 t_0.2-0.4=1.0162 t_0.4-0.6=0.8303 t_0.6-0.8=0.7492 t_0.8-1.0=0.8037] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0005 (BEST) + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 6/299 (2% done) --- + [Epoch 6][10/66] loss=0.889533 avg=0.887488 VRAM=38.7GiB | 2.1% done | ETA(epoch): 1077s + [Epoch 6][20/66] loss=0.929415 avg=0.893249 VRAM=38.6GiB | 2.1% done | ETA(epoch): 886s + [Epoch 6][30/66] loss=0.837748 avg=0.880907 VRAM=38.7GiB | 2.2% done | ETA(epoch): 694s + [Epoch 6][40/66] loss=0.842090 avg=0.885214 VRAM=38.6GiB | 2.2% done | ETA(epoch): 501s + [Epoch 6][50/66] loss=0.996590 avg=0.886290 VRAM=38.7GiB | 2.3% done | ETA(epoch): 308s + [Epoch 6][60/66] loss=0.994836 avg=0.890356 VRAM=38.6GiB | 2.3% done | ETA(epoch): 116s + Train loss: 0.887013 (1271.8s) ETA: 7320min + Val loss: 0.879437 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9793 t_0.4-0.6=0.8294 t_0.6-0.8=0.7559 t_0.8-1.0=0.7854] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0006 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 7/299 (2% done) --- + [Epoch 7][10/66] loss=1.007792 avg=0.894619 VRAM=38.7GiB | 2.4% done | ETA(epoch): 1088s + [Epoch 7][20/66] loss=0.847033 avg=0.894303 VRAM=38.6GiB | 2.4% done | ETA(epoch): 890s + [Epoch 7][30/66] loss=0.800524 avg=0.897100 VRAM=38.7GiB | 2.5% done | ETA(epoch): 695s + [Epoch 7][40/66] loss=0.850841 avg=0.899684 VRAM=38.6GiB | 2.5% done | ETA(epoch): 502s + [Epoch 7][50/66] loss=0.735125 avg=0.892565 VRAM=38.7GiB | 2.6% done | ETA(epoch): 309s + [Epoch 7][60/66] loss=0.941802 avg=0.891199 VRAM=38.6GiB | 2.6% done | ETA(epoch): 116s + Train loss: 0.892666 (1273.0s) ETA: 7216min + Val loss: 0.891807 [t_0.0-0.2=1.0549 t_0.2-0.4=1.0348 t_0.4-0.6=0.8784 t_0.6-0.8=0.7459 t_0.8-1.0=0.7602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 8/299 (3% done) --- + [Epoch 8][10/66] loss=0.984413 avg=0.899358 VRAM=38.7GiB | 2.7% done | ETA(epoch): 1077s + [Epoch 8][20/66] loss=0.967338 avg=0.889266 VRAM=38.6GiB | 2.8% done | ETA(epoch): 885s + [Epoch 8][30/66] loss=0.831910 avg=0.887982 VRAM=38.7GiB | 2.8% done | ETA(epoch): 693s + [Epoch 8][40/66] loss=0.990462 avg=0.898972 VRAM=38.6GiB | 2.9% done | ETA(epoch): 501s + [Epoch 8][50/66] loss=0.855176 avg=0.895848 VRAM=38.7GiB | 2.9% done | ETA(epoch): 308s + [Epoch 8][60/66] loss=0.874795 avg=0.894705 VRAM=38.6GiB | 3.0% done | ETA(epoch): 116s + Train loss: 0.890402 (1271.3s) ETA: 7130min + Val loss: 0.903192 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0123 t_0.4-0.6=0.8449 t_0.6-0.8=0.7611 t_0.8-1.0=0.7689] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 9/299 (3% done) --- + [Epoch 9][10/66] loss=0.807423 avg=0.866653 VRAM=38.7GiB | 3.1% done | ETA(epoch): 1078s + [Epoch 9][20/66] loss=0.791603 avg=0.873449 VRAM=38.6GiB | 3.1% done | ETA(epoch): 886s + [Epoch 9][30/66] loss=0.907419 avg=0.883651 VRAM=38.7GiB | 3.2% done | ETA(epoch): 693s + [Epoch 9][40/66] loss=0.913629 avg=0.893140 VRAM=38.6GiB | 3.2% done | ETA(epoch): 501s + [Epoch 9][50/66] loss=0.794450 avg=0.884031 VRAM=38.7GiB | 3.3% done | ETA(epoch): 308s + [Epoch 9][60/66] loss=0.941124 avg=0.885932 VRAM=38.6GiB | 3.3% done | ETA(epoch): 116s + Train loss: 0.885059 (1271.4s) ETA: 7055min + Val loss: 0.901717 [t_0.0-0.2=1.0626 t_0.2-0.4=0.9905 t_0.4-0.6=0.8657 t_0.6-0.8=0.7575 t_0.8-1.0=0.7616] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0009 +[MEM @ epoch 9 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 10/299 (3% done) --- + [Epoch 10][10/66] loss=0.818717 avg=0.879775 VRAM=38.7GiB | 3.4% done | ETA(epoch): 1080s + [Epoch 10][20/66] loss=0.778497 avg=0.873199 VRAM=38.6GiB | 3.4% done | ETA(epoch): 887s + [Epoch 10][30/66] loss=0.810585 avg=0.874639 VRAM=38.7GiB | 3.5% done | ETA(epoch): 694s + [Epoch 10][40/66] loss=0.788622 avg=0.878395 VRAM=38.6GiB | 3.5% done | ETA(epoch): 501s + [Epoch 10][50/66] loss=0.822612 avg=0.881643 VRAM=38.7GiB | 3.6% done | ETA(epoch): 308s + [Epoch 10][60/66] loss=0.838074 avg=0.879088 VRAM=38.6GiB | 3.6% done | ETA(epoch): 116s + Train loss: 0.875833 (1271.4s) ETA: 6990min + Val loss: 0.877179 [t_0.0-0.2=1.0900 t_0.2-0.4=1.0080 t_0.4-0.6=0.8474 t_0.6-0.8=0.7427 t_0.8-1.0=0.7402] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0010 (BEST) + Deleted old checkpoint: checkpoint_epoch_0006 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 11/299 (4% done) --- + [Epoch 11][10/66] loss=0.856650 avg=0.909726 VRAM=38.7GiB | 3.7% done | ETA(epoch): 1080s + [Epoch 11][20/66] loss=0.949534 avg=0.902644 VRAM=38.6GiB | 3.8% done | ETA(epoch): 886s + [Epoch 11][30/66] loss=0.886562 avg=0.892980 VRAM=38.7GiB | 3.8% done | ETA(epoch): 693s + [Epoch 11][40/66] loss=0.827948 avg=0.889444 VRAM=38.6GiB | 3.9% done | ETA(epoch): 501s + [Epoch 11][50/66] loss=0.851329 avg=0.884748 VRAM=38.7GiB | 3.9% done | ETA(epoch): 308s + [Epoch 11][60/66] loss=0.886812 avg=0.882928 VRAM=38.6GiB | 4.0% done | ETA(epoch): 116s + Train loss: 0.886759 (1270.7s) ETA: 6932min + Val loss: 0.853299 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9767 t_0.4-0.6=0.8087 t_0.6-0.8=0.7547 t_0.8-1.0=0.7566] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0011 (BEST) + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 12/299 (4% done) --- + [Epoch 12][10/66] loss=0.923366 avg=0.875145 VRAM=38.7GiB | 4.1% done | ETA(epoch): 1077s + [Epoch 12][20/66] loss=0.937119 avg=0.875271 VRAM=38.6GiB | 4.1% done | ETA(epoch): 885s + [Epoch 12][30/66] loss=0.880023 avg=0.868456 VRAM=38.7GiB | 4.2% done | ETA(epoch): 697s + [Epoch 12][40/66] loss=0.829426 avg=0.866370 VRAM=38.6GiB | 4.2% done | ETA(epoch): 502s + [Epoch 12][50/66] loss=0.996343 avg=0.866485 VRAM=38.7GiB | 4.3% done | ETA(epoch): 309s + [Epoch 12][60/66] loss=0.828035 avg=0.865732 VRAM=38.6GiB | 4.3% done | ETA(epoch): 116s + Train loss: 0.864647 (1272.5s) ETA: 6880min + Val loss: 0.891816 [t_0.0-0.2=1.0672 t_0.2-0.4=1.0211 t_0.4-0.6=0.8477 t_0.6-0.8=0.7397 t_0.8-1.0=0.7519] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 13/299 (4% done) --- + [Epoch 13][10/66] loss=0.776739 avg=0.879563 VRAM=38.7GiB | 4.4% done | ETA(epoch): 1079s + [Epoch 13][20/66] loss=0.846821 avg=0.858908 VRAM=38.6GiB | 4.4% done | ETA(epoch): 890s + [Epoch 13][30/66] loss=0.909563 avg=0.876808 VRAM=38.7GiB | 4.5% done | ETA(epoch): 696s + [Epoch 13][40/66] loss=0.869317 avg=0.875363 VRAM=38.6GiB | 4.5% done | ETA(epoch): 501s + [Epoch 13][50/66] loss=0.817304 avg=0.876206 VRAM=38.7GiB | 4.6% done | ETA(epoch): 308s + [Epoch 13][60/66] loss=0.770912 avg=0.869330 VRAM=38.6GiB | 4.6% done | ETA(epoch): 116s + Train loss: 0.873809 (1271.4s) ETA: 6831min + Val loss: 0.882470 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0076 t_0.4-0.6=0.8680 t_0.6-0.8=0.7190 t_0.8-1.0=0.7449] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 14/299 (5% done) --- + [Epoch 14][10/66] loss=0.916051 avg=0.877706 VRAM=38.7GiB | 4.7% done | ETA(epoch): 1076s + [Epoch 14][20/66] loss=0.943182 avg=0.892759 VRAM=38.6GiB | 4.8% done | ETA(epoch): 889s + [Epoch 14][30/66] loss=0.972372 avg=0.891430 VRAM=38.7GiB | 4.8% done | ETA(epoch): 694s + [Epoch 14][40/66] loss=0.946988 avg=0.885028 VRAM=38.6GiB | 4.9% done | ETA(epoch): 501s + [Epoch 14][50/66] loss=0.843846 avg=0.881852 VRAM=38.7GiB | 4.9% done | ETA(epoch): 308s + [Epoch 14][60/66] loss=0.864854 avg=0.885532 VRAM=38.6GiB | 5.0% done | ETA(epoch): 116s + Train loss: 0.882252 (1271.8s) ETA: 6786min + Val loss: 0.865201 [t_0.0-0.2=1.0773 t_0.2-0.4=0.9722 t_0.4-0.6=0.8676 t_0.6-0.8=0.7436 t_0.8-1.0=0.7267] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0014 +[MEM @ epoch 14 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 15/299 (5% done) --- + [Epoch 15][10/66] loss=0.932412 avg=0.899684 VRAM=38.7GiB | 5.1% done | ETA(epoch): 1096s + [Epoch 15][20/66] loss=0.760710 avg=0.896718 VRAM=38.6GiB | 5.1% done | ETA(epoch): 893s + [Epoch 15][30/66] loss=0.873878 avg=0.884504 VRAM=38.7GiB | 5.2% done | ETA(epoch): 696s + [Epoch 15][40/66] loss=1.010499 avg=0.887609 VRAM=38.6GiB | 5.2% done | ETA(epoch): 502s + [Epoch 15][50/66] loss=0.868148 avg=0.888226 VRAM=38.7GiB | 5.3% done | ETA(epoch): 309s + [Epoch 15][60/66] loss=0.978417 avg=0.891553 VRAM=38.6GiB | 5.3% done | ETA(epoch): 116s + Train loss: 0.887850 (1272.5s) ETA: 6745min + Val loss: 0.852138 [t_0.0-0.2=1.0659 t_0.2-0.4=0.9860 t_0.4-0.6=0.8791 t_0.6-0.8=0.7272 t_0.8-1.0=0.7195] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0015 (BEST) + Deleted old checkpoint: checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 16/299 (5% done) --- + [Epoch 16][10/66] loss=0.839584 avg=0.875239 VRAM=38.7GiB | 5.4% done | ETA(epoch): 1076s + [Epoch 16][20/66] loss=0.873915 avg=0.887109 VRAM=38.6GiB | 5.4% done | ETA(epoch): 884s + [Epoch 16][30/66] loss=0.832711 avg=0.886172 VRAM=38.7GiB | 5.5% done | ETA(epoch): 692s + [Epoch 16][40/66] loss=0.965951 avg=0.883102 VRAM=38.6GiB | 5.5% done | ETA(epoch): 500s + [Epoch 16][50/66] loss=0.807490 avg=0.879373 VRAM=38.7GiB | 5.6% done | ETA(epoch): 308s + [Epoch 16][60/66] loss=0.917980 avg=0.876048 VRAM=38.6GiB | 5.6% done | ETA(epoch): 115s + Train loss: 0.877614 (1269.4s) ETA: 6704min + Val loss: 0.902386 [t_0.0-0.2=1.0802 t_0.2-0.4=1.0254 t_0.4-0.6=0.8529 t_0.6-0.8=0.7411 t_0.8-1.0=0.7333] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 17/299 (6% done) --- + [Epoch 17][10/66] loss=0.724764 avg=0.862340 VRAM=38.7GiB | 5.7% done | ETA(epoch): 1076s + [Epoch 17][20/66] loss=0.791333 avg=0.861153 VRAM=38.6GiB | 5.8% done | ETA(epoch): 885s + [Epoch 17][30/66] loss=0.865173 avg=0.868261 VRAM=38.7GiB | 5.8% done | ETA(epoch): 692s + [Epoch 17][40/66] loss=0.930043 avg=0.879819 VRAM=38.6GiB | 5.9% done | ETA(epoch): 500s + [Epoch 17][50/66] loss=0.914103 avg=0.882226 VRAM=38.7GiB | 5.9% done | ETA(epoch): 308s + [Epoch 17][60/66] loss=0.889246 avg=0.886350 VRAM=38.6GiB | 6.0% done | ETA(epoch): 115s + Train loss: 0.882275 (1269.5s) ETA: 6666min + Val loss: 0.889376 [t_0.0-0.2=1.0805 t_0.2-0.4=1.0066 t_0.4-0.6=0.8804 t_0.6-0.8=0.7110 t_0.8-1.0=0.7094] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 18/299 (6% done) --- + [Epoch 18][10/66] loss=0.837197 avg=0.890424 VRAM=38.7GiB | 6.1% done | ETA(epoch): 1077s + [Epoch 18][20/66] loss=0.916753 avg=0.875331 VRAM=38.6GiB | 6.1% done | ETA(epoch): 885s + [Epoch 18][30/66] loss=0.752711 avg=0.878907 VRAM=38.7GiB | 6.2% done | ETA(epoch): 692s + [Epoch 18][40/66] loss=0.804433 avg=0.883962 VRAM=38.6GiB | 6.2% done | ETA(epoch): 500s + [Epoch 18][50/66] loss=0.869913 avg=0.882262 VRAM=38.7GiB | 6.3% done | ETA(epoch): 308s + [Epoch 18][60/66] loss=0.848073 avg=0.880678 VRAM=38.6GiB | 6.3% done | ETA(epoch): 115s + Train loss: 0.880677 (1269.7s) ETA: 6629min + Val loss: 0.899518 [t_0.0-0.2=1.0798 t_0.2-0.4=0.9775 t_0.4-0.6=0.8400 t_0.6-0.8=0.7323 t_0.8-1.0=0.7151] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0018 +[MEM @ epoch 18 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 19/299 (6% done) --- + [Epoch 19][10/66] loss=0.893114 avg=0.851207 VRAM=38.7GiB | 6.4% done | ETA(epoch): 1076s + [Epoch 19][20/66] loss=0.934958 avg=0.853537 VRAM=38.6GiB | 6.4% done | ETA(epoch): 884s + [Epoch 19][30/66] loss=0.855394 avg=0.863215 VRAM=38.7GiB | 6.5% done | ETA(epoch): 692s + [Epoch 19][40/66] loss=0.890733 avg=0.864930 VRAM=38.6GiB | 6.5% done | ETA(epoch): 500s + [Epoch 19][50/66] loss=0.901161 avg=0.870340 VRAM=38.7GiB | 6.6% done | ETA(epoch): 308s + [Epoch 19][60/66] loss=0.858472 avg=0.871953 VRAM=38.6GiB | 6.6% done | ETA(epoch): 115s + Train loss: 0.871194 (1268.4s) ETA: 6593min + Val loss: 0.850663 [t_0.0-0.2=1.0821 t_0.2-0.4=0.9925 t_0.4-0.6=0.8809 t_0.6-0.8=0.7029 t_0.8-1.0=0.7206] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0019 (BEST) + Deleted old checkpoint: checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 20/299 (7% done) --- + [Epoch 20][10/66] loss=0.903568 avg=0.821765 VRAM=38.7GiB | 6.7% done | ETA(epoch): 1076s + [Epoch 20][20/66] loss=0.870017 avg=0.851726 VRAM=38.6GiB | 6.8% done | ETA(epoch): 884s + [Epoch 20][30/66] loss=0.818313 avg=0.848669 VRAM=38.7GiB | 6.8% done | ETA(epoch): 692s + [Epoch 20][40/66] loss=0.965108 avg=0.854213 VRAM=38.6GiB | 6.9% done | ETA(epoch): 500s + [Epoch 20][50/66] loss=0.777283 avg=0.861759 VRAM=38.7GiB | 6.9% done | ETA(epoch): 308s + [Epoch 20][60/66] loss=0.997348 avg=0.867813 VRAM=38.6GiB | 7.0% done | ETA(epoch): 115s + Train loss: 0.866928 (1269.6s) ETA: 6559min + Val loss: 0.859882 [t_0.0-0.2=1.0739 t_0.2-0.4=1.0086 t_0.4-0.6=0.8515 t_0.6-0.8=0.7393 t_0.8-1.0=0.7153] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 21/299 (7% done) --- + [Epoch 21][10/66] loss=0.900239 avg=0.838705 VRAM=38.7GiB | 7.1% done | ETA(epoch): 1078s + [Epoch 21][20/66] loss=0.826738 avg=0.847394 VRAM=38.6GiB | 7.1% done | ETA(epoch): 886s + [Epoch 21][30/66] loss=0.850350 avg=0.862556 VRAM=38.7GiB | 7.2% done | ETA(epoch): 693s + [Epoch 21][40/66] loss=0.989240 avg=0.866175 VRAM=38.6GiB | 7.2% done | ETA(epoch): 500s + [Epoch 21][50/66] loss=0.893815 avg=0.869995 VRAM=38.7GiB | 7.3% done | ETA(epoch): 308s + [Epoch 21][60/66] loss=0.876490 avg=0.871838 VRAM=38.6GiB | 7.3% done | ETA(epoch): 115s + Train loss: 0.871438 (1270.2s) ETA: 6526min + Val loss: 0.842528 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9247 t_0.4-0.6=0.8509 t_0.6-0.8=0.7430 t_0.8-1.0=0.7110] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0021 (BEST) + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 22/299 (7% done) --- + [Epoch 22][10/66] loss=0.710632 avg=0.881590 VRAM=38.7GiB | 7.4% done | ETA(epoch): 1075s + [Epoch 22][20/66] loss=0.930723 avg=0.867518 VRAM=38.6GiB | 7.4% done | ETA(epoch): 884s + [Epoch 22][30/66] loss=0.823472 avg=0.868787 VRAM=38.7GiB | 7.5% done | ETA(epoch): 692s + [Epoch 22][40/66] loss=0.947556 avg=0.869391 VRAM=38.6GiB | 7.5% done | ETA(epoch): 500s + [Epoch 22][50/66] loss=0.849307 avg=0.865973 VRAM=38.7GiB | 7.6% done | ETA(epoch): 308s + [Epoch 22][60/66] loss=0.907831 avg=0.868622 VRAM=38.6GiB | 7.6% done | ETA(epoch): 115s + Train loss: 0.868764 (1269.5s) ETA: 6494min + Val loss: 0.871167 [t_0.0-0.2=1.0807 t_0.2-0.4=0.9972 t_0.4-0.6=0.8589 t_0.6-0.8=0.7333 t_0.8-1.0=0.7172] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 23/299 (8% done) --- + [Epoch 23][10/66] loss=0.947867 avg=0.891939 VRAM=38.7GiB | 7.7% done | ETA(epoch): 1076s + [Epoch 23][20/66] loss=0.854231 avg=0.890585 VRAM=38.6GiB | 7.8% done | ETA(epoch): 884s + [Epoch 23][30/66] loss=0.974306 avg=0.892008 VRAM=38.7GiB | 7.8% done | ETA(epoch): 692s + [Epoch 23][40/66] loss=0.793579 avg=0.889364 VRAM=38.6GiB | 7.9% done | ETA(epoch): 500s + [Epoch 23][50/66] loss=0.941279 avg=0.890880 VRAM=38.7GiB | 7.9% done | ETA(epoch): 308s + [Epoch 23][60/66] loss=0.933099 avg=0.887476 VRAM=38.6GiB | 8.0% done | ETA(epoch): 115s + Train loss: 0.886359 (1268.8s) ETA: 6463min + Val loss: 0.868818 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0001 t_0.4-0.6=0.8027 t_0.6-0.8=0.7162 t_0.8-1.0=0.7208] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 24/299 (8% done) --- + [Epoch 24][10/66] loss=0.845080 avg=0.883019 VRAM=38.7GiB | 8.1% done | ETA(epoch): 1077s + [Epoch 24][20/66] loss=0.824124 avg=0.871702 VRAM=38.6GiB | 8.1% done | ETA(epoch): 885s + [Epoch 24][30/66] loss=0.988884 avg=0.880992 VRAM=38.7GiB | 8.2% done | ETA(epoch): 692s + [Epoch 24][40/66] loss=0.852048 avg=0.878211 VRAM=38.6GiB | 8.2% done | ETA(epoch): 500s + [Epoch 24][50/66] loss=0.970829 avg=0.882704 VRAM=38.7GiB | 8.3% done | ETA(epoch): 308s + [Epoch 24][60/66] loss=0.883840 avg=0.881208 VRAM=38.6GiB | 8.3% done | ETA(epoch): 116s + Train loss: 0.881548 (1271.6s) ETA: 6433min + Val loss: 0.877819 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9919 t_0.4-0.6=0.8769 t_0.6-0.8=0.7151 t_0.8-1.0=0.6916] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0024 +[MEM @ epoch 24 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 25/299 (8% done) --- + [Epoch 25][10/66] loss=0.972317 avg=0.863165 VRAM=38.7GiB | 8.4% done | ETA(epoch): 1076s + [Epoch 25][20/66] loss=0.796682 avg=0.872708 VRAM=38.6GiB | 8.4% done | ETA(epoch): 884s + [Epoch 25][30/66] loss=0.891493 avg=0.878759 VRAM=38.7GiB | 8.5% done | ETA(epoch): 693s + [Epoch 25][40/66] loss=0.920619 avg=0.877171 VRAM=38.6GiB | 8.5% done | ETA(epoch): 500s + [Epoch 25][50/66] loss=0.843226 avg=0.880913 VRAM=38.7GiB | 8.6% done | ETA(epoch): 308s + [Epoch 25][60/66] loss=0.851944 avg=0.878137 VRAM=38.6GiB | 8.6% done | ETA(epoch): 116s + Train loss: 0.879496 (1271.6s) ETA: 6403min + Val loss: 0.894972 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0079 t_0.4-0.6=0.8575 t_0.6-0.8=0.7407 t_0.8-1.0=0.7291] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 26/299 (9% done) --- + [Epoch 26][10/66] loss=0.737092 avg=0.882164 VRAM=38.7GiB | 8.7% done | ETA(epoch): 1075s + [Epoch 26][20/66] loss=0.910713 avg=0.887906 VRAM=38.6GiB | 8.8% done | ETA(epoch): 884s + [Epoch 26][30/66] loss=0.957518 avg=0.886812 VRAM=38.7GiB | 8.8% done | ETA(epoch): 692s + [Epoch 26][40/66] loss=0.884717 avg=0.886812 VRAM=38.6GiB | 8.9% done | ETA(epoch): 500s + [Epoch 26][50/66] loss=0.964137 avg=0.886429 VRAM=38.7GiB | 8.9% done | ETA(epoch): 308s + [Epoch 26][60/66] loss=0.919806 avg=0.880929 VRAM=38.6GiB | 9.0% done | ETA(epoch): 115s + Train loss: 0.879471 (1269.7s) ETA: 6373min + Val loss: 0.844671 [t_0.0-0.2=1.0756 t_0.2-0.4=1.0203 t_0.4-0.6=0.8570 t_0.6-0.8=0.7307 t_0.8-1.0=0.7024] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 27/299 (9% done) --- + [Epoch 27][10/66] loss=0.865480 avg=0.867293 VRAM=38.7GiB | 9.1% done | ETA(epoch): 1076s + [Epoch 27][20/66] loss=0.855582 avg=0.864583 VRAM=38.6GiB | 9.1% done | ETA(epoch): 884s + [Epoch 27][30/66] loss=0.944595 avg=0.876546 VRAM=38.7GiB | 9.2% done | ETA(epoch): 692s + [Epoch 27][40/66] loss=0.933015 avg=0.875681 VRAM=38.6GiB | 9.2% done | ETA(epoch): 500s + [Epoch 27][50/66] loss=0.892853 avg=0.876785 VRAM=38.7GiB | 9.3% done | ETA(epoch): 308s + [Epoch 27][60/66] loss=0.843397 avg=0.870034 VRAM=38.6GiB | 9.3% done | ETA(epoch): 115s + Train loss: 0.865974 (1269.7s) ETA: 6344min + Val loss: 0.883712 [t_0.0-0.2=1.0734 t_0.2-0.4=1.0067 t_0.4-0.6=0.8136 t_0.6-0.8=0.7478 t_0.8-1.0=0.6930] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 28/299 (9% done) --- + [Epoch 28][10/66] loss=0.907131 avg=0.853670 VRAM=38.7GiB | 9.4% done | ETA(epoch): 1078s + [Epoch 28][20/66] loss=0.865141 avg=0.868027 VRAM=38.6GiB | 9.4% done | ETA(epoch): 885s + [Epoch 28][30/66] loss=0.957054 avg=0.856159 VRAM=38.7GiB | 9.5% done | ETA(epoch): 693s + [Epoch 28][40/66] loss=0.933171 avg=0.861641 VRAM=38.6GiB | 9.5% done | ETA(epoch): 501s + [Epoch 28][50/66] loss=0.953561 avg=0.862482 VRAM=38.7GiB | 9.6% done | ETA(epoch): 308s + [Epoch 28][60/66] loss=0.869572 avg=0.866096 VRAM=38.6GiB | 9.6% done | ETA(epoch): 116s + Train loss: 0.866992 (1270.5s) ETA: 6315min + Val loss: 0.849337 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9866 t_0.4-0.6=0.9001 t_0.6-0.8=0.7346 t_0.8-1.0=0.6812] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 29/299 (10% done) --- + [Epoch 29][10/66] loss=0.788389 avg=0.869621 VRAM=38.7GiB | 9.7% done | ETA(epoch): 1076s + [Epoch 29][20/66] loss=0.752659 avg=0.883994 VRAM=38.6GiB | 9.8% done | ETA(epoch): 885s + [Epoch 29][30/66] loss=0.761239 avg=0.871636 VRAM=38.7GiB | 9.8% done | ETA(epoch): 693s + [Epoch 29][40/66] loss=0.980570 avg=0.871669 VRAM=38.6GiB | 9.9% done | ETA(epoch): 500s + [Epoch 29][50/66] loss=0.864238 avg=0.864235 VRAM=38.7GiB | 9.9% done | ETA(epoch): 308s + [Epoch 29][60/66] loss=0.857725 avg=0.863335 VRAM=38.6GiB | 10.0% done | ETA(epoch): 115s + Train loss: 0.861784 (1269.1s) ETA: 6287min + Val loss: 0.867931 [t_0.0-0.2=1.0709 t_0.2-0.4=1.0125 t_0.4-0.6=0.8217 t_0.6-0.8=0.7261 t_0.8-1.0=0.6775] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 30/299 (10% done) --- + [MilestoneVis] train_0 step 1980 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 1980 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 1980 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 1980 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 1980 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 1980 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 1980 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 1980 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 1980 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 1980 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 30][10/66] loss=0.884425 avg=0.905279 VRAM=38.7GiB | 10.1% done | ETA(epoch): 6807s + [Epoch 30][20/66] loss=0.969780 avg=0.906040 VRAM=38.6GiB | 10.1% done | ETA(epoch): 3244s + [Epoch 30][30/66] loss=0.798725 avg=0.892955 VRAM=38.7GiB | 10.2% done | ETA(epoch): 1923s + [Epoch 30][40/66] loss=0.889121 avg=0.878046 VRAM=38.6GiB | 10.2% done | ETA(epoch): 1167s + [Epoch 30][50/66] loss=0.806799 avg=0.880574 VRAM=38.7GiB | 10.3% done | ETA(epoch): 636s + [Epoch 30][60/66] loss=0.783486 avg=0.870495 VRAM=38.6GiB | 10.3% done | ETA(epoch): 218s + Train loss: 0.874437 (2295.6s) ETA: 6407min + Val loss: 0.860710 [t_0.0-0.2=1.0702 t_0.2-0.4=1.0221 t_0.4-0.6=0.8632 t_0.6-0.8=0.7154 t_0.8-1.0=0.6938] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 31/299 (10% done) --- + [Epoch 31][10/66] loss=0.760449 avg=0.868203 VRAM=38.7GiB | 10.4% done | ETA(epoch): 1078s + [Epoch 31][20/66] loss=0.833617 avg=0.882905 VRAM=38.6GiB | 10.4% done | ETA(epoch): 885s + [Epoch 31][30/66] loss=0.878376 avg=0.886400 VRAM=38.7GiB | 10.5% done | ETA(epoch): 692s + [Epoch 31][40/66] loss=0.903940 avg=0.880291 VRAM=38.6GiB | 10.5% done | ETA(epoch): 500s + [Epoch 31][50/66] loss=0.845762 avg=0.885785 VRAM=38.7GiB | 10.6% done | ETA(epoch): 308s + [Epoch 31][60/66] loss=0.916436 avg=0.883611 VRAM=38.6GiB | 10.6% done | ETA(epoch): 115s + Train loss: 0.879251 (1269.7s) ETA: 6375min + Val loss: 0.847415 [t_0.0-0.2=1.0850 t_0.2-0.4=1.0236 t_0.4-0.6=0.8311 t_0.6-0.8=0.7203 t_0.8-1.0=0.6931] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 32/299 (11% done) --- + [Epoch 32][10/66] loss=0.912626 avg=0.891068 VRAM=38.7GiB | 10.7% done | ETA(epoch): 1077s + [Epoch 32][20/66] loss=0.909465 avg=0.881951 VRAM=38.6GiB | 10.8% done | ETA(epoch): 885s + [Epoch 32][30/66] loss=0.885473 avg=0.884474 VRAM=38.7GiB | 10.8% done | ETA(epoch): 692s + [Epoch 32][40/66] loss=0.766945 avg=0.881385 VRAM=38.6GiB | 10.9% done | ETA(epoch): 500s + [Epoch 32][50/66] loss=1.025402 avg=0.887238 VRAM=38.7GiB | 10.9% done | ETA(epoch): 308s + [Epoch 32][60/66] loss=0.990121 avg=0.883414 VRAM=38.6GiB | 11.0% done | ETA(epoch): 115s + Train loss: 0.885572 (1269.5s) ETA: 6342min + Val loss: 0.838042 [t_0.0-0.2=1.0627 t_0.2-0.4=0.9968 t_0.4-0.6=0.8543 t_0.6-0.8=0.7339 t_0.8-1.0=0.6929] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0032 (BEST) + Deleted old checkpoint: checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 19.5/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 33/299 (11% done) --- + [Epoch 33][10/66] loss=0.888611 avg=0.884155 VRAM=38.7GiB | 11.1% done | ETA(epoch): 1076s + [Epoch 33][20/66] loss=0.932503 avg=0.882458 VRAM=38.6GiB | 11.1% done | ETA(epoch): 884s + [Epoch 33][30/66] loss=0.905205 avg=0.883351 VRAM=38.7GiB | 11.2% done | ETA(epoch): 692s + [Epoch 33][40/66] loss=0.912111 avg=0.881587 VRAM=38.6GiB | 11.2% done | ETA(epoch): 500s + [Epoch 33][50/66] loss=0.941059 avg=0.878185 VRAM=38.7GiB | 11.3% done | ETA(epoch): 308s + [Epoch 33][60/66] loss=0.854011 avg=0.874718 VRAM=38.6GiB | 11.3% done | ETA(epoch): 115s + Train loss: 0.871692 (1269.7s) ETA: 6311min + Val loss: 0.863222 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0126 t_0.4-0.6=0.8924 t_0.6-0.8=0.7372 t_0.8-1.0=0.6775] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 34/299 (11% done) --- + [Epoch 34][10/66] loss=0.916917 avg=0.883650 VRAM=38.7GiB | 11.4% done | ETA(epoch): 1077s + [Epoch 34][20/66] loss=0.896459 avg=0.887385 VRAM=38.6GiB | 11.4% done | ETA(epoch): 886s + [Epoch 34][30/66] loss=0.802722 avg=0.883946 VRAM=38.7GiB | 11.5% done | ETA(epoch): 693s + [Epoch 34][40/66] loss=0.880147 avg=0.872979 VRAM=38.6GiB | 11.5% done | ETA(epoch): 500s + [Epoch 34][50/66] loss=0.854509 avg=0.875702 VRAM=38.7GiB | 11.6% done | ETA(epoch): 308s + [Epoch 34][60/66] loss=0.834640 avg=0.876835 VRAM=38.6GiB | 11.6% done | ETA(epoch): 116s + Train loss: 0.875124 (1271.0s) ETA: 6280min + Val loss: 0.840910 [t_0.0-0.2=1.0987 t_0.2-0.4=0.9910 t_0.4-0.6=0.8426 t_0.6-0.8=0.7163 t_0.8-1.0=0.6745] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 35/299 (12% done) --- + [Epoch 35][10/66] loss=0.813171 avg=0.882032 VRAM=38.7GiB | 11.7% done | ETA(epoch): 1077s + [Epoch 35][20/66] loss=0.820703 avg=0.849764 VRAM=38.6GiB | 11.8% done | ETA(epoch): 886s + [Epoch 35][30/66] loss=0.831041 avg=0.863028 VRAM=38.7GiB | 11.8% done | ETA(epoch): 693s + [Epoch 35][40/66] loss=0.972508 avg=0.867131 VRAM=38.6GiB | 11.9% done | ETA(epoch): 502s + [Epoch 35][50/66] loss=0.794583 avg=0.872150 VRAM=38.7GiB | 11.9% done | ETA(epoch): 309s + [Epoch 35][60/66] loss=0.808102 avg=0.869557 VRAM=38.6GiB | 12.0% done | ETA(epoch): 116s + Train loss: 0.867468 (1272.9s) ETA: 6249min + Val loss: 0.874006 [t_0.0-0.2=1.0590 t_0.2-0.4=0.9975 t_0.4-0.6=0.8956 t_0.6-0.8=0.7105 t_0.8-1.0=0.6841] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0035 +[MEM @ epoch 35 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 36/299 (12% done) --- + [Epoch 36][10/66] loss=0.775997 avg=0.858814 VRAM=38.7GiB | 12.1% done | ETA(epoch): 1078s + [Epoch 36][20/66] loss=0.877830 avg=0.859600 VRAM=38.6GiB | 12.1% done | ETA(epoch): 885s + [Epoch 36][30/66] loss=0.969794 avg=0.875188 VRAM=38.7GiB | 12.2% done | ETA(epoch): 693s + [Epoch 36][40/66] loss=0.783953 avg=0.866214 VRAM=38.6GiB | 12.2% done | ETA(epoch): 501s + [Epoch 36][50/66] loss=0.929659 avg=0.867773 VRAM=38.7GiB | 12.3% done | ETA(epoch): 308s + [Epoch 36][60/66] loss=0.911507 avg=0.864055 VRAM=38.6GiB | 12.3% done | ETA(epoch): 116s + Train loss: 0.865301 (1271.2s) ETA: 6219min + Val loss: 0.851005 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0009 t_0.4-0.6=0.8597 t_0.6-0.8=0.7100 t_0.8-1.0=0.6862] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 37/299 (12% done) --- + [Epoch 37][10/66] loss=0.974746 avg=0.838863 VRAM=38.7GiB | 12.4% done | ETA(epoch): 1076s + [Epoch 37][20/66] loss=0.730672 avg=0.859161 VRAM=38.6GiB | 12.4% done | ETA(epoch): 884s + [Epoch 37][30/66] loss=0.862386 avg=0.855851 VRAM=38.7GiB | 12.5% done | ETA(epoch): 692s + [Epoch 37][40/66] loss=0.766706 avg=0.851452 VRAM=38.6GiB | 12.5% done | ETA(epoch): 500s + [Epoch 37][50/66] loss=0.847124 avg=0.846674 VRAM=38.7GiB | 12.6% done | ETA(epoch): 308s + [Epoch 37][60/66] loss=0.889009 avg=0.845910 VRAM=38.6GiB | 12.6% done | ETA(epoch): 115s + Train loss: 0.845978 (1269.2s) ETA: 6189min + Val loss: 0.863015 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9968 t_0.4-0.6=0.8449 t_0.6-0.8=0.7243 t_0.8-1.0=0.6954] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 38/299 (13% done) --- + [Epoch 38][10/66] loss=0.783078 avg=0.844263 VRAM=38.7GiB | 12.7% done | ETA(epoch): 1078s + [Epoch 38][20/66] loss=0.922807 avg=0.850094 VRAM=38.6GiB | 12.8% done | ETA(epoch): 885s + [Epoch 38][30/66] loss=0.865414 avg=0.860554 VRAM=38.7GiB | 12.8% done | ETA(epoch): 693s + [Epoch 38][40/66] loss=1.001537 avg=0.860855 VRAM=38.6GiB | 12.9% done | ETA(epoch): 501s + [Epoch 38][50/66] loss=0.887393 avg=0.861106 VRAM=38.7GiB | 12.9% done | ETA(epoch): 308s + [Epoch 38][60/66] loss=0.936230 avg=0.861682 VRAM=38.6GiB | 13.0% done | ETA(epoch): 116s + Train loss: 0.860625 (1270.7s) ETA: 6160min + Val loss: 0.847379 [t_0.0-0.2=1.0770 t_0.2-0.4=0.9787 t_0.4-0.6=0.8405 t_0.6-0.8=0.7447 t_0.8-1.0=0.6782] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 39/299 (13% done) --- + [Epoch 39][10/66] loss=0.870016 avg=0.862030 VRAM=38.7GiB | 13.1% done | ETA(epoch): 1079s + [Epoch 39][20/66] loss=0.782604 avg=0.856604 VRAM=38.6GiB | 13.1% done | ETA(epoch): 886s + [Epoch 39][30/66] loss=0.963964 avg=0.868365 VRAM=38.7GiB | 13.2% done | ETA(epoch): 693s + [Epoch 39][40/66] loss=0.778906 avg=0.860625 VRAM=38.6GiB | 13.2% done | ETA(epoch): 501s + [Epoch 39][50/66] loss=0.974998 avg=0.869290 VRAM=38.7GiB | 13.3% done | ETA(epoch): 308s + [Epoch 39][60/66] loss=0.737077 avg=0.865056 VRAM=38.6GiB | 13.3% done | ETA(epoch): 115s + Train loss: 0.863074 (1270.5s) ETA: 6131min + Val loss: 0.894089 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9632 t_0.4-0.6=0.8582 t_0.6-0.8=0.7419 t_0.8-1.0=0.6675] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 40/299 (13% done) --- + [Epoch 40][10/66] loss=0.945928 avg=0.906413 VRAM=38.7GiB | 13.4% done | ETA(epoch): 1077s + [Epoch 40][20/66] loss=0.821238 avg=0.902649 VRAM=38.6GiB | 13.4% done | ETA(epoch): 886s + [Epoch 40][30/66] loss=0.862399 avg=0.888947 VRAM=38.7GiB | 13.5% done | ETA(epoch): 694s + [Epoch 40][40/66] loss=0.869945 avg=0.875475 VRAM=38.6GiB | 13.5% done | ETA(epoch): 501s + [Epoch 40][50/66] loss=0.828056 avg=0.873298 VRAM=38.7GiB | 13.6% done | ETA(epoch): 308s + [Epoch 40][60/66] loss=0.909898 avg=0.880356 VRAM=38.6GiB | 13.6% done | ETA(epoch): 116s + Train loss: 0.878826 (1273.1s) ETA: 6102min + Val loss: 0.848590 [t_0.0-0.2=1.0864 t_0.2-0.4=0.9855 t_0.4-0.6=0.8372 t_0.6-0.8=0.7391 t_0.8-1.0=0.6786] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 41/299 (14% done) --- + [Epoch 41][10/66] loss=0.835537 avg=0.892545 VRAM=38.7GiB | 13.7% done | ETA(epoch): 1077s + [Epoch 41][20/66] loss=0.822478 avg=0.864383 VRAM=38.6GiB | 13.8% done | ETA(epoch): 885s + [Epoch 41][30/66] loss=0.742150 avg=0.857706 VRAM=38.7GiB | 13.8% done | ETA(epoch): 693s + [Epoch 41][40/66] loss=0.951247 avg=0.865851 VRAM=38.6GiB | 13.9% done | ETA(epoch): 501s + [Epoch 41][50/66] loss=0.957553 avg=0.863935 VRAM=38.7GiB | 13.9% done | ETA(epoch): 308s + [Epoch 41][60/66] loss=0.806437 avg=0.865193 VRAM=38.6GiB | 14.0% done | ETA(epoch): 115s + Train loss: 0.864840 (1270.0s) ETA: 6074min + Val loss: 0.848363 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9918 t_0.4-0.6=0.8424 t_0.6-0.8=0.7230 t_0.8-1.0=0.6668] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 18.9/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 42/299 (14% done) --- + [Epoch 42][10/66] loss=0.865123 avg=0.875499 VRAM=38.7GiB | 14.1% done | ETA(epoch): 1076s + [Epoch 42][20/66] loss=1.000747 avg=0.872725 VRAM=38.6GiB | 14.1% done | ETA(epoch): 884s + [Epoch 42][30/66] loss=0.840349 avg=0.866841 VRAM=38.7GiB | 14.2% done | ETA(epoch): 693s + [Epoch 42][40/66] loss=0.898146 avg=0.872407 VRAM=38.6GiB | 14.2% done | ETA(epoch): 502s + [Epoch 42][50/66] loss=0.727823 avg=0.875216 VRAM=38.7GiB | 14.3% done | ETA(epoch): 309s + [Epoch 42][60/66] loss=0.805595 avg=0.870678 VRAM=38.6GiB | 14.3% done | ETA(epoch): 116s + Train loss: 0.871699 (1272.5s) ETA: 6046min + Val loss: 0.831032 [t_0.0-0.2=1.0874 t_0.2-0.4=0.9978 t_0.4-0.6=0.8567 t_0.6-0.8=0.7239 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0042 (BEST) + Deleted old checkpoint: checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 43/299 (14% done) --- + [Epoch 43][10/66] loss=0.824401 avg=0.853581 VRAM=38.7GiB | 14.4% done | ETA(epoch): 1075s + [Epoch 43][20/66] loss=0.783473 avg=0.858295 VRAM=38.6GiB | 14.4% done | ETA(epoch): 884s + [Epoch 43][30/66] loss=0.971378 avg=0.866428 VRAM=38.7GiB | 14.5% done | ETA(epoch): 692s + [Epoch 43][40/66] loss=0.709895 avg=0.859954 VRAM=38.6GiB | 14.5% done | ETA(epoch): 500s + [Epoch 43][50/66] loss=0.945221 avg=0.865424 VRAM=38.7GiB | 14.6% done | ETA(epoch): 308s + [Epoch 43][60/66] loss=0.946425 avg=0.858666 VRAM=38.6GiB | 14.6% done | ETA(epoch): 115s + Train loss: 0.864482 (1269.8s) ETA: 6018min + Val loss: 0.885423 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9751 t_0.4-0.6=0.8904 t_0.6-0.8=0.6982 t_0.8-1.0=0.6710] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 44/299 (15% done) --- + [Epoch 44][10/66] loss=1.003111 avg=0.862830 VRAM=38.7GiB | 14.7% done | ETA(epoch): 1078s + [Epoch 44][20/66] loss=0.760244 avg=0.850591 VRAM=38.6GiB | 14.8% done | ETA(epoch): 886s + [Epoch 44][30/66] loss=0.931751 avg=0.851606 VRAM=38.7GiB | 14.8% done | ETA(epoch): 693s + [Epoch 44][40/66] loss=1.003587 avg=0.862035 VRAM=38.6GiB | 14.9% done | ETA(epoch): 501s + [Epoch 44][50/66] loss=0.791933 avg=0.868368 VRAM=38.7GiB | 14.9% done | ETA(epoch): 308s + [Epoch 44][60/66] loss=0.840676 avg=0.867028 VRAM=38.6GiB | 15.0% done | ETA(epoch): 116s + Train loss: 0.870048 (1270.9s) ETA: 5990min + Val loss: 0.869633 [t_0.0-0.2=1.0567 t_0.2-0.4=0.9706 t_0.4-0.6=0.8702 t_0.6-0.8=0.7061 t_0.8-1.0=0.6857] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 45/299 (15% done) --- + [Epoch 45][10/66] loss=0.876314 avg=0.823851 VRAM=38.7GiB | 15.1% done | ETA(epoch): 1087s + [Epoch 45][20/66] loss=0.839035 avg=0.827111 VRAM=38.6GiB | 15.1% done | ETA(epoch): 889s + [Epoch 45][30/66] loss=0.886505 avg=0.834530 VRAM=38.7GiB | 15.2% done | ETA(epoch): 695s + [Epoch 45][40/66] loss=1.025467 avg=0.854600 VRAM=38.6GiB | 15.2% done | ETA(epoch): 502s + [Epoch 45][50/66] loss=0.890408 avg=0.860433 VRAM=38.7GiB | 15.3% done | ETA(epoch): 308s + [Epoch 45][60/66] loss=0.908192 avg=0.857278 VRAM=38.6GiB | 15.3% done | ETA(epoch): 116s + Train loss: 0.857364 (1271.6s) ETA: 5963min + Val loss: 0.844574 [t_0.0-0.2=1.0927 t_0.2-0.4=1.0014 t_0.4-0.6=0.7710 t_0.6-0.8=0.7285 t_0.8-1.0=0.6560] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0045 +[MEM @ epoch 45 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 46/299 (15% done) --- + [Epoch 46][10/66] loss=0.899845 avg=0.897328 VRAM=38.7GiB | 15.4% done | ETA(epoch): 1077s + [Epoch 46][20/66] loss=0.920454 avg=0.899353 VRAM=38.6GiB | 15.4% done | ETA(epoch): 885s + [Epoch 46][30/66] loss=0.785645 avg=0.883766 VRAM=38.7GiB | 15.5% done | ETA(epoch): 693s + [Epoch 46][40/66] loss=0.951901 avg=0.881179 VRAM=38.6GiB | 15.5% done | ETA(epoch): 500s + [Epoch 46][50/66] loss=0.968659 avg=0.876344 VRAM=38.7GiB | 15.6% done | ETA(epoch): 308s + [Epoch 46][60/66] loss=0.890451 avg=0.868711 VRAM=38.6GiB | 15.6% done | ETA(epoch): 115s + Train loss: 0.867581 (1270.5s) ETA: 5936min + Val loss: 0.855299 [t_0.0-0.2=1.0634 t_0.2-0.4=0.9952 t_0.4-0.6=0.8604 t_0.6-0.8=0.7441 t_0.8-1.0=0.6719] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 47/299 (16% done) --- + [Epoch 47][10/66] loss=0.834137 avg=0.880383 VRAM=38.7GiB | 15.7% done | ETA(epoch): 1078s + [Epoch 47][20/66] loss=0.828840 avg=0.874851 VRAM=38.6GiB | 15.8% done | ETA(epoch): 885s + [Epoch 47][30/66] loss=0.971112 avg=0.885180 VRAM=38.7GiB | 15.8% done | ETA(epoch): 693s + [Epoch 47][40/66] loss=0.882694 avg=0.879683 VRAM=38.6GiB | 15.9% done | ETA(epoch): 500s + [Epoch 47][50/66] loss=0.770910 avg=0.877923 VRAM=38.7GiB | 15.9% done | ETA(epoch): 308s + [Epoch 47][60/66] loss=0.849860 avg=0.876243 VRAM=38.6GiB | 16.0% done | ETA(epoch): 116s + Train loss: 0.869960 (1270.9s) ETA: 5908min + Val loss: 0.839181 [t_0.0-0.2=1.0732 t_0.2-0.4=1.0430 t_0.4-0.6=0.8381 t_0.6-0.8=0.7034 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 48/299 (16% done) --- + [Epoch 48][10/66] loss=0.802799 avg=0.852966 VRAM=38.7GiB | 16.1% done | ETA(epoch): 1076s + [Epoch 48][20/66] loss=0.801301 avg=0.864358 VRAM=38.6GiB | 16.1% done | ETA(epoch): 885s + [Epoch 48][30/66] loss=0.849850 avg=0.865225 VRAM=38.7GiB | 16.2% done | ETA(epoch): 692s + [Epoch 48][40/66] loss=0.946426 avg=0.863103 VRAM=38.6GiB | 16.2% done | ETA(epoch): 500s + [Epoch 48][50/66] loss=0.765375 avg=0.857915 VRAM=38.7GiB | 16.3% done | ETA(epoch): 314s + [Epoch 48][60/66] loss=0.819871 avg=0.856119 VRAM=38.6GiB | 16.3% done | ETA(epoch): 117s + Train loss: 0.856915 (1287.8s) ETA: 5883min + Val loss: 0.872795 [t_0.0-0.2=1.0895 t_0.2-0.4=1.0219 t_0.4-0.6=0.8555 t_0.6-0.8=0.7355 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 19.4/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 49/299 (16% done) --- + [Epoch 49][10/66] loss=0.854359 avg=0.880775 VRAM=38.7GiB | 16.4% done | ETA(epoch): 1077s + [Epoch 49][20/66] loss=0.709925 avg=0.865383 VRAM=38.6GiB | 16.4% done | ETA(epoch): 885s + [Epoch 49][30/66] loss=0.964525 avg=0.866749 VRAM=38.7GiB | 16.5% done | ETA(epoch): 693s + [Epoch 49][40/66] loss=0.990060 avg=0.863193 VRAM=38.6GiB | 16.5% done | ETA(epoch): 500s + [Epoch 49][50/66] loss=0.836257 avg=0.857851 VRAM=38.7GiB | 16.6% done | ETA(epoch): 308s + [Epoch 49][60/66] loss=0.796590 avg=0.856469 VRAM=38.6GiB | 16.6% done | ETA(epoch): 115s + Train loss: 0.855136 (1269.7s) ETA: 5856min + Val loss: 0.866288 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9998 t_0.4-0.6=0.8505 t_0.6-0.8=0.7254 t_0.8-1.0=0.6752] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 50/299 (17% done) --- + [Epoch 50][10/66] loss=0.941054 avg=0.895049 VRAM=38.7GiB | 16.7% done | ETA(epoch): 1077s + [Epoch 50][20/66] loss=0.795341 avg=0.870521 VRAM=38.6GiB | 16.8% done | ETA(epoch): 885s + [Epoch 50][30/66] loss=0.779538 avg=0.868885 VRAM=38.7GiB | 16.8% done | ETA(epoch): 693s + [Epoch 50][40/66] loss=0.868154 avg=0.868095 VRAM=38.6GiB | 16.9% done | ETA(epoch): 501s + [Epoch 50][50/66] loss=0.853037 avg=0.873077 VRAM=38.7GiB | 16.9% done | ETA(epoch): 308s + [Epoch 50][60/66] loss=0.718780 avg=0.867109 VRAM=38.6GiB | 17.0% done | ETA(epoch): 115s + Train loss: 0.865646 (1270.3s) ETA: 5829min + Val loss: 0.828462 [t_0.0-0.2=1.0719 t_0.2-0.4=0.9887 t_0.4-0.6=0.8449 t_0.6-0.8=0.7336 t_0.8-1.0=0.6806] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0050 (BEST) + Deleted old checkpoint: checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 51/299 (17% done) --- + [Epoch 51][10/66] loss=0.780199 avg=0.832193 VRAM=38.7GiB | 17.1% done | ETA(epoch): 1077s + [Epoch 51][20/66] loss=0.908852 avg=0.856266 VRAM=38.6GiB | 17.1% done | ETA(epoch): 885s + [Epoch 51][30/66] loss=0.832762 avg=0.851540 VRAM=38.7GiB | 17.2% done | ETA(epoch): 692s + [Epoch 51][40/66] loss=0.858365 avg=0.865820 VRAM=38.6GiB | 17.2% done | ETA(epoch): 500s + [Epoch 51][50/66] loss=0.870828 avg=0.857469 VRAM=38.7GiB | 17.3% done | ETA(epoch): 308s + [Epoch 51][60/66] loss=0.832738 avg=0.859169 VRAM=38.6GiB | 17.3% done | ETA(epoch): 115s + Train loss: 0.861995 (1269.7s) ETA: 5803min + Val loss: 0.862244 [t_0.0-0.2=1.0577 t_0.2-0.4=1.0229 t_0.4-0.6=0.8069 t_0.6-0.8=0.7417 t_0.8-1.0=0.6764] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 52/299 (17% done) --- + [Epoch 52][10/66] loss=0.985605 avg=0.856762 VRAM=38.7GiB | 17.4% done | ETA(epoch): 1078s + [Epoch 52][20/66] loss=0.836705 avg=0.860141 VRAM=38.6GiB | 17.4% done | ETA(epoch): 891s + [Epoch 52][30/66] loss=0.795315 avg=0.864094 VRAM=38.7GiB | 17.5% done | ETA(epoch): 696s + [Epoch 52][40/66] loss=0.827862 avg=0.866239 VRAM=38.6GiB | 17.5% done | ETA(epoch): 502s + [Epoch 52][50/66] loss=0.832743 avg=0.864636 VRAM=38.7GiB | 17.6% done | ETA(epoch): 309s + [Epoch 52][60/66] loss=0.816108 avg=0.861671 VRAM=38.6GiB | 17.6% done | ETA(epoch): 116s + Train loss: 0.863665 (1273.6s) ETA: 5777min + Val loss: 0.872662 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9898 t_0.4-0.6=0.8639 t_0.6-0.8=0.7527 t_0.8-1.0=0.6736] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 53/299 (18% done) --- + [Epoch 53][10/66] loss=0.726901 avg=0.846719 VRAM=38.7GiB | 17.7% done | ETA(epoch): 1088s + [Epoch 53][20/66] loss=0.975186 avg=0.887363 VRAM=38.6GiB | 17.8% done | ETA(epoch): 889s + [Epoch 53][30/66] loss=0.924833 avg=0.894647 VRAM=38.7GiB | 17.8% done | ETA(epoch): 695s + [Epoch 53][40/66] loss=0.821693 avg=0.887079 VRAM=38.6GiB | 17.9% done | ETA(epoch): 502s + [Epoch 53][50/66] loss=0.930168 avg=0.883333 VRAM=38.7GiB | 17.9% done | ETA(epoch): 309s + [Epoch 53][60/66] loss=0.943530 avg=0.881196 VRAM=38.6GiB | 18.0% done | ETA(epoch): 116s + Train loss: 0.875666 (1272.6s) ETA: 5751min + Val loss: 0.880313 [t_0.0-0.2=1.0569 t_0.2-0.4=1.0359 t_0.4-0.6=0.8949 t_0.6-0.8=0.7162 t_0.8-1.0=0.6682] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0053 +[MEM @ epoch 53 end] RAM: 18.4/188.5 GiB (9.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 54/299 (18% done) --- + [Epoch 54][10/66] loss=0.899863 avg=0.862677 VRAM=38.7GiB | 18.1% done | ETA(epoch): 1077s + [Epoch 54][20/66] loss=0.896571 avg=0.863927 VRAM=38.6GiB | 18.1% done | ETA(epoch): 886s + [Epoch 54][30/66] loss=1.023769 avg=0.854941 VRAM=38.7GiB | 18.2% done | ETA(epoch): 693s + [Epoch 54][40/66] loss=0.815146 avg=0.853353 VRAM=38.6GiB | 18.2% done | ETA(epoch): 501s + [Epoch 54][50/66] loss=0.842042 avg=0.852271 VRAM=38.7GiB | 18.3% done | ETA(epoch): 308s + [Epoch 54][60/66] loss=0.809314 avg=0.856695 VRAM=38.6GiB | 18.3% done | ETA(epoch): 116s + Train loss: 0.859389 (1270.6s) ETA: 5724min + Val loss: 0.871899 [t_0.0-0.2=1.0699 t_0.2-0.4=1.0087 t_0.4-0.6=0.8192 t_0.6-0.8=0.7262 t_0.8-1.0=0.6724] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 55/299 (18% done) --- + [Epoch 55][10/66] loss=0.900193 avg=0.850853 VRAM=38.7GiB | 18.4% done | ETA(epoch): 1077s + [Epoch 55][20/66] loss=0.818003 avg=0.876623 VRAM=38.6GiB | 18.4% done | ETA(epoch): 885s + [Epoch 55][30/66] loss=0.745577 avg=0.867755 VRAM=38.7GiB | 18.5% done | ETA(epoch): 692s + [Epoch 55][40/66] loss=0.829273 avg=0.870909 VRAM=38.6GiB | 18.5% done | ETA(epoch): 500s + [Epoch 55][50/66] loss=0.835825 avg=0.871862 VRAM=38.7GiB | 18.6% done | ETA(epoch): 308s + [Epoch 55][60/66] loss=0.744943 avg=0.866036 VRAM=38.6GiB | 18.6% done | ETA(epoch): 116s + Train loss: 0.862157 (1272.7s) ETA: 5699min + Val loss: 0.832167 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9681 t_0.4-0.6=0.8646 t_0.6-0.8=0.7281 t_0.8-1.0=0.6510] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 56/299 (19% done) --- + [Epoch 56][10/66] loss=0.804572 avg=0.843902 VRAM=38.7GiB | 18.7% done | ETA(epoch): 1077s + [Epoch 56][20/66] loss=1.023020 avg=0.852795 VRAM=38.6GiB | 18.8% done | ETA(epoch): 885s + [Epoch 56][30/66] loss=0.866054 avg=0.859278 VRAM=38.7GiB | 18.8% done | ETA(epoch): 693s + [Epoch 56][40/66] loss=0.767221 avg=0.855503 VRAM=38.6GiB | 18.9% done | ETA(epoch): 500s + [Epoch 56][50/66] loss=0.859547 avg=0.854365 VRAM=38.7GiB | 18.9% done | ETA(epoch): 308s + [Epoch 56][60/66] loss=0.848153 avg=0.857633 VRAM=38.6GiB | 19.0% done | ETA(epoch): 116s + Train loss: 0.858984 (1271.6s) ETA: 5673min + Val loss: 0.835374 [t_0.0-0.2=1.0825 t_0.2-0.4=1.0093 t_0.4-0.6=0.8448 t_0.6-0.8=0.7118 t_0.8-1.0=0.6660] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 57/299 (19% done) --- + [Epoch 57][10/66] loss=0.903999 avg=0.887422 VRAM=38.7GiB | 19.1% done | ETA(epoch): 1079s + [Epoch 57][20/66] loss=0.949385 avg=0.877620 VRAM=38.6GiB | 19.1% done | ETA(epoch): 886s + [Epoch 57][30/66] loss=0.912311 avg=0.871816 VRAM=38.7GiB | 19.2% done | ETA(epoch): 693s + [Epoch 57][40/66] loss=0.820575 avg=0.872672 VRAM=38.6GiB | 19.2% done | ETA(epoch): 500s + [Epoch 57][50/66] loss=0.808867 avg=0.872127 VRAM=38.7GiB | 19.3% done | ETA(epoch): 308s + [Epoch 57][60/66] loss=0.839187 avg=0.869507 VRAM=38.6GiB | 19.3% done | ETA(epoch): 115s + Train loss: 0.870155 (1269.5s) ETA: 5647min + Val loss: 0.874471 [t_0.0-0.2=1.0956 t_0.2-0.4=0.9886 t_0.4-0.6=0.8449 t_0.6-0.8=0.7041 t_0.8-1.0=0.6785] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 58/299 (19% done) --- + [Epoch 58][10/66] loss=0.918948 avg=0.861940 VRAM=38.7GiB | 19.4% done | ETA(epoch): 1075s + [Epoch 58][20/66] loss=0.794621 avg=0.873390 VRAM=38.6GiB | 19.4% done | ETA(epoch): 884s + [Epoch 58][30/66] loss=0.869971 avg=0.871447 VRAM=38.7GiB | 19.5% done | ETA(epoch): 692s + [Epoch 58][40/66] loss=0.953210 avg=0.874379 VRAM=38.6GiB | 19.5% done | ETA(epoch): 500s + [Epoch 58][50/66] loss=0.779447 avg=0.871432 VRAM=38.7GiB | 19.6% done | ETA(epoch): 307s + [Epoch 58][60/66] loss=0.835049 avg=0.882563 VRAM=38.6GiB | 19.6% done | ETA(epoch): 115s + Train loss: 0.881654 (1268.5s) ETA: 5621min + Val loss: 0.853108 [t_0.0-0.2=1.0551 t_0.2-0.4=1.0244 t_0.4-0.6=0.8371 t_0.6-0.8=0.7225 t_0.8-1.0=0.6580] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 59/299 (20% done) --- + [Epoch 59][10/66] loss=0.791811 avg=0.865715 VRAM=38.7GiB | 19.7% done | ETA(epoch): 1077s + [Epoch 59][20/66] loss=0.938470 avg=0.878653 VRAM=38.6GiB | 19.8% done | ETA(epoch): 889s + [Epoch 59][30/66] loss=0.784425 avg=0.871644 VRAM=38.7GiB | 19.8% done | ETA(epoch): 695s + [Epoch 59][40/66] loss=0.755391 avg=0.862041 VRAM=38.6GiB | 19.9% done | ETA(epoch): 501s + [Epoch 59][50/66] loss=0.922165 avg=0.862695 VRAM=38.7GiB | 19.9% done | ETA(epoch): 308s + [Epoch 59][60/66] loss=1.012689 avg=0.871540 VRAM=38.6GiB | 20.0% done | ETA(epoch): 116s + Train loss: 0.873254 (1271.5s) ETA: 5596min + Val loss: 0.885119 [t_0.0-0.2=1.0906 t_0.2-0.4=0.9846 t_0.4-0.6=0.8345 t_0.6-0.8=0.7555 t_0.8-1.0=0.7239] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 60/299 (20% done) --- + [MilestoneVis] train_0 step 3960 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3960 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3960 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3960 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3960 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3960 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3960 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3960 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3960 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3960 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 60][10/66] loss=0.738567 avg=0.866982 VRAM=38.7GiB | 20.1% done | ETA(epoch): 6788s + [Epoch 60][20/66] loss=0.945127 avg=0.855446 VRAM=38.6GiB | 20.1% done | ETA(epoch): 3230s + [Epoch 60][30/66] loss=0.974476 avg=0.857427 VRAM=38.7GiB | 20.2% done | ETA(epoch): 1916s + [Epoch 60][40/66] loss=0.879421 avg=0.864695 VRAM=38.6GiB | 20.2% done | ETA(epoch): 1163s + [Epoch 60][50/66] loss=0.881496 avg=0.864036 VRAM=38.7GiB | 20.3% done | ETA(epoch): 634s + [Epoch 60][60/66] loss=0.823659 avg=0.866769 VRAM=38.6GiB | 20.3% done | ETA(epoch): 217s + Train loss: 0.864427 (2289.3s) ETA: 5637min + Val loss: 0.876558 [t_0.0-0.2=1.0851 t_0.2-0.4=0.9743 t_0.4-0.6=0.8096 t_0.6-0.8=0.7417 t_0.8-1.0=0.6678] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 61/299 (20% done) --- + [Epoch 61][10/66] loss=0.797466 avg=0.870058 VRAM=38.7GiB | 20.4% done | ETA(epoch): 1077s + [Epoch 61][20/66] loss=0.896138 avg=0.874580 VRAM=38.6GiB | 20.4% done | ETA(epoch): 891s + [Epoch 61][30/66] loss=0.982956 avg=0.887469 VRAM=38.7GiB | 20.5% done | ETA(epoch): 696s + [Epoch 61][40/66] loss=0.870854 avg=0.878745 VRAM=38.6GiB | 20.5% done | ETA(epoch): 502s + [Epoch 61][50/66] loss=0.954906 avg=0.878294 VRAM=38.7GiB | 20.6% done | ETA(epoch): 309s + [Epoch 61][60/66] loss=0.966298 avg=0.878979 VRAM=38.6GiB | 20.6% done | ETA(epoch): 116s + Train loss: 0.878752 (1272.3s) ETA: 5610min + Val loss: 0.881016 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9966 t_0.4-0.6=0.8406 t_0.6-0.8=0.7454 t_0.8-1.0=0.6863] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 62/299 (21% done) --- + [Epoch 62][10/66] loss=0.834560 avg=0.870513 VRAM=38.7GiB | 20.7% done | ETA(epoch): 1076s + [Epoch 62][20/66] loss=0.837794 avg=0.852860 VRAM=38.6GiB | 20.8% done | ETA(epoch): 884s + [Epoch 62][30/66] loss=0.817476 avg=0.849449 VRAM=38.7GiB | 20.8% done | ETA(epoch): 692s + [Epoch 62][40/66] loss=0.928207 avg=0.860735 VRAM=38.6GiB | 20.9% done | ETA(epoch): 500s + [Epoch 62][50/66] loss=0.808046 avg=0.869399 VRAM=38.7GiB | 20.9% done | ETA(epoch): 308s + [Epoch 62][60/66] loss=0.734097 avg=0.865028 VRAM=38.6GiB | 21.0% done | ETA(epoch): 115s + Train loss: 0.869989 (1271.5s) ETA: 5583min + Val loss: 0.869568 [t_0.0-0.2=1.0790 t_0.2-0.4=1.0200 t_0.4-0.6=0.8681 t_0.6-0.8=0.7492 t_0.8-1.0=0.7092] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 63/299 (21% done) --- + [Epoch 63][10/66] loss=0.814997 avg=0.861362 VRAM=38.7GiB | 21.1% done | ETA(epoch): 1077s + [Epoch 63][20/66] loss=0.942825 avg=0.863492 VRAM=38.6GiB | 21.1% done | ETA(epoch): 884s + [Epoch 63][30/66] loss=0.968741 avg=0.875658 VRAM=38.7GiB | 21.2% done | ETA(epoch): 692s + [Epoch 63][40/66] loss=0.863312 avg=0.883899 VRAM=38.6GiB | 21.2% done | ETA(epoch): 500s + [Epoch 63][50/66] loss=0.931591 avg=0.885084 VRAM=38.7GiB | 21.3% done | ETA(epoch): 308s + [Epoch 63][60/66] loss=0.816447 avg=0.885744 VRAM=38.6GiB | 21.3% done | ETA(epoch): 115s + Train loss: 0.885398 (1268.9s) ETA: 5557min + Val loss: 0.901580 [t_0.0-0.2=1.0870 t_0.2-0.4=1.0097 t_0.4-0.6=0.8401 t_0.6-0.8=0.7297 t_0.8-1.0=0.6927] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 64/299 (21% done) --- + [Epoch 64][10/66] loss=0.871387 avg=0.876068 VRAM=38.7GiB | 21.4% done | ETA(epoch): 1075s + [Epoch 64][20/66] loss=0.934077 avg=0.884484 VRAM=38.6GiB | 21.4% done | ETA(epoch): 884s + [Epoch 64][30/66] loss=0.977062 avg=0.889343 VRAM=38.7GiB | 21.5% done | ETA(epoch): 692s + [Epoch 64][40/66] loss=0.922428 avg=0.875288 VRAM=38.6GiB | 21.5% done | ETA(epoch): 500s + [Epoch 64][50/66] loss=0.828015 avg=0.870147 VRAM=38.7GiB | 21.6% done | ETA(epoch): 309s + [Epoch 64][60/66] loss=0.828576 avg=0.866758 VRAM=38.6GiB | 21.6% done | ETA(epoch): 116s + Train loss: 0.867653 (1271.6s) ETA: 5530min + Val loss: 0.904507 [t_0.0-0.2=1.0827 t_0.2-0.4=0.9947 t_0.4-0.6=0.8673 t_0.6-0.8=0.7354 t_0.8-1.0=0.6722] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 65/299 (22% done) --- + [Epoch 65][10/66] loss=0.827862 avg=0.847838 VRAM=38.7GiB | 21.7% done | ETA(epoch): 1076s + [Epoch 65][20/66] loss=0.787536 avg=0.853681 VRAM=38.6GiB | 21.8% done | ETA(epoch): 884s + [Epoch 65][30/66] loss=0.756832 avg=0.854002 VRAM=38.7GiB | 21.8% done | ETA(epoch): 692s + [Epoch 65][40/66] loss=0.751822 avg=0.854712 VRAM=38.6GiB | 21.9% done | ETA(epoch): 500s + [Epoch 65][50/66] loss=0.947470 avg=0.855022 VRAM=38.7GiB | 21.9% done | ETA(epoch): 308s + [Epoch 65][60/66] loss=0.933171 avg=0.862199 VRAM=38.6GiB | 22.0% done | ETA(epoch): 115s + Train loss: 0.865428 (1269.0s) ETA: 5504min + Val loss: 0.886220 [t_0.0-0.2=1.0824 t_0.2-0.4=0.9875 t_0.4-0.6=0.8400 t_0.6-0.8=0.7145 t_0.8-1.0=0.6730] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 66/299 (22% done) --- + [Epoch 66][10/66] loss=0.879295 avg=0.863341 VRAM=38.7GiB | 22.1% done | ETA(epoch): 1076s + [Epoch 66][20/66] loss=0.820807 avg=0.847751 VRAM=38.6GiB | 22.1% done | ETA(epoch): 885s + [Epoch 66][30/66] loss=0.788207 avg=0.853645 VRAM=38.7GiB | 22.2% done | ETA(epoch): 693s + [Epoch 66][40/66] loss=0.771600 avg=0.853928 VRAM=38.6GiB | 22.2% done | ETA(epoch): 500s + [Epoch 66][50/66] loss=0.947684 avg=0.855410 VRAM=38.7GiB | 22.3% done | ETA(epoch): 308s + [Epoch 66][60/66] loss=0.873169 avg=0.860014 VRAM=38.6GiB | 22.3% done | ETA(epoch): 115s + Train loss: 0.859977 (1269.6s) ETA: 5478min + Val loss: 0.872971 [t_0.0-0.2=1.0781 t_0.2-0.4=1.0008 t_0.4-0.6=0.8305 t_0.6-0.8=0.7146 t_0.8-1.0=0.6597] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 67/299 (22% done) --- + [Epoch 67][10/66] loss=0.812912 avg=0.817395 VRAM=38.7GiB | 22.4% done | ETA(epoch): 1076s + [Epoch 67][20/66] loss=0.946600 avg=0.852307 VRAM=38.6GiB | 22.4% done | ETA(epoch): 885s + [Epoch 67][30/66] loss=0.843191 avg=0.868835 VRAM=38.7GiB | 22.5% done | ETA(epoch): 696s + [Epoch 67][40/66] loss=0.828769 avg=0.863215 VRAM=38.6GiB | 22.5% done | ETA(epoch): 502s + [Epoch 67][50/66] loss=0.912813 avg=0.857988 VRAM=38.7GiB | 22.6% done | ETA(epoch): 309s + [Epoch 67][60/66] loss=0.962461 avg=0.866152 VRAM=38.6GiB | 22.6% done | ETA(epoch): 116s + Train loss: 0.865001 (1272.3s) ETA: 5452min + Val loss: 0.863588 [t_0.0-0.2=1.0840 t_0.2-0.4=1.0174 t_0.4-0.6=0.8839 t_0.6-0.8=0.6909 t_0.8-1.0=0.6712] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 68/299 (23% done) --- + [Epoch 68][10/66] loss=0.879576 avg=0.887520 VRAM=38.7GiB | 22.7% done | ETA(epoch): 1079s + [Epoch 68][20/66] loss=0.838413 avg=0.862305 VRAM=38.6GiB | 22.8% done | ETA(epoch): 894s + [Epoch 68][30/66] loss=0.751089 avg=0.864388 VRAM=38.7GiB | 22.8% done | ETA(epoch): 697s + [Epoch 68][40/66] loss=0.802494 avg=0.864899 VRAM=38.6GiB | 22.9% done | ETA(epoch): 503s + [Epoch 68][50/66] loss=0.961741 avg=0.862424 VRAM=38.7GiB | 22.9% done | ETA(epoch): 309s + [Epoch 68][60/66] loss=0.911101 avg=0.860241 VRAM=38.6GiB | 23.0% done | ETA(epoch): 116s + Train loss: 0.855414 (1273.8s) ETA: 5426min + Val loss: 0.902222 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9847 t_0.4-0.6=0.8594 t_0.6-0.8=0.7378 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 69/299 (23% done) --- + [Epoch 69][10/66] loss=0.825566 avg=0.847992 VRAM=38.7GiB | 23.1% done | ETA(epoch): 1088s + [Epoch 69][20/66] loss=0.866260 avg=0.832097 VRAM=38.6GiB | 23.1% done | ETA(epoch): 889s + [Epoch 69][30/66] loss=0.894633 avg=0.853216 VRAM=38.7GiB | 23.2% done | ETA(epoch): 695s + [Epoch 69][40/66] loss=0.763925 avg=0.846395 VRAM=38.6GiB | 23.2% done | ETA(epoch): 502s + [Epoch 69][50/66] loss=0.859609 avg=0.842492 VRAM=38.7GiB | 23.3% done | ETA(epoch): 308s + [Epoch 69][60/66] loss=0.870558 avg=0.843434 VRAM=38.6GiB | 23.3% done | ETA(epoch): 116s + Train loss: 0.839921 (1271.5s) ETA: 5400min + Val loss: 0.858354 [t_0.0-0.2=1.0717 t_0.2-0.4=1.0265 t_0.4-0.6=0.8616 t_0.6-0.8=0.7111 t_0.8-1.0=0.6644] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 70/299 (23% done) --- + [Epoch 70][10/66] loss=0.988185 avg=0.871180 VRAM=38.7GiB | 23.4% done | ETA(epoch): 1075s + [Epoch 70][20/66] loss=0.992854 avg=0.861088 VRAM=38.6GiB | 23.4% done | ETA(epoch): 884s + [Epoch 70][30/66] loss=0.976837 avg=0.870503 VRAM=38.7GiB | 23.5% done | ETA(epoch): 691s + [Epoch 70][40/66] loss=0.872543 avg=0.869211 VRAM=38.6GiB | 23.5% done | ETA(epoch): 500s + [Epoch 70][50/66] loss=0.878766 avg=0.864700 VRAM=38.7GiB | 23.6% done | ETA(epoch): 307s + [Epoch 70][60/66] loss=0.953961 avg=0.861539 VRAM=38.6GiB | 23.6% done | ETA(epoch): 115s + Train loss: 0.862872 (1270.4s) ETA: 5374min + Val loss: 0.869600 [t_0.0-0.2=1.0591 t_0.2-0.4=0.9774 t_0.4-0.6=0.8628 t_0.6-0.8=0.7043 t_0.8-1.0=0.6754] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 71/299 (24% done) --- + [Epoch 71][10/66] loss=0.935934 avg=0.869364 VRAM=38.7GiB | 23.7% done | ETA(epoch): 1075s + [Epoch 71][20/66] loss=0.862215 avg=0.867337 VRAM=38.6GiB | 23.8% done | ETA(epoch): 883s + [Epoch 71][30/66] loss=0.838125 avg=0.868049 VRAM=38.7GiB | 23.8% done | ETA(epoch): 691s + [Epoch 71][40/66] loss=0.940471 avg=0.859701 VRAM=38.6GiB | 23.9% done | ETA(epoch): 500s + [Epoch 71][50/66] loss=0.994460 avg=0.862927 VRAM=38.7GiB | 23.9% done | ETA(epoch): 308s + [Epoch 71][60/66] loss=0.866770 avg=0.862585 VRAM=38.6GiB | 24.0% done | ETA(epoch): 116s + Train loss: 0.862759 (1270.5s) ETA: 5349min + Val loss: 0.874374 [t_0.0-0.2=1.0805 t_0.2-0.4=1.0064 t_0.4-0.6=0.8225 t_0.6-0.8=0.7314 t_0.8-1.0=0.6737] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 72/299 (24% done) --- + [Epoch 72][10/66] loss=0.861008 avg=0.880197 VRAM=38.7GiB | 24.1% done | ETA(epoch): 1076s + [Epoch 72][20/66] loss=0.975805 avg=0.862456 VRAM=38.6GiB | 24.1% done | ETA(epoch): 884s + [Epoch 72][30/66] loss=0.820870 avg=0.860265 VRAM=38.7GiB | 24.2% done | ETA(epoch): 692s + [Epoch 72][40/66] loss=0.925401 avg=0.863037 VRAM=38.6GiB | 24.2% done | ETA(epoch): 500s + [Epoch 72][50/66] loss=0.856332 avg=0.865938 VRAM=38.7GiB | 24.3% done | ETA(epoch): 309s + [Epoch 72][60/66] loss=0.934936 avg=0.869965 VRAM=38.6GiB | 24.3% done | ETA(epoch): 116s + Train loss: 0.869203 (1272.9s) ETA: 5323min + Val loss: 0.847792 [t_0.0-0.2=1.0786 t_0.2-0.4=0.9847 t_0.4-0.6=0.8688 t_0.6-0.8=0.7395 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 73/299 (24% done) --- + [Epoch 73][10/66] loss=0.741829 avg=0.834901 VRAM=38.7GiB | 24.4% done | ETA(epoch): 1074s + [Epoch 73][20/66] loss=0.895536 avg=0.834492 VRAM=38.6GiB | 24.4% done | ETA(epoch): 883s + [Epoch 73][30/66] loss=0.857618 avg=0.846059 VRAM=38.7GiB | 24.5% done | ETA(epoch): 691s + [Epoch 73][40/66] loss=0.936665 avg=0.848717 VRAM=38.6GiB | 24.5% done | ETA(epoch): 502s + [Epoch 73][50/66] loss=0.913613 avg=0.846574 VRAM=38.7GiB | 24.6% done | ETA(epoch): 309s + [Epoch 73][60/66] loss=0.899269 avg=0.845670 VRAM=38.6GiB | 24.6% done | ETA(epoch): 116s + Train loss: 0.850074 (1272.8s) ETA: 5298min + Val loss: 0.859507 [t_0.0-0.2=1.0756 t_0.2-0.4=1.0133 t_0.4-0.6=0.8770 t_0.6-0.8=0.7128 t_0.8-1.0=0.6527] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 74/299 (25% done) --- + [Epoch 74][10/66] loss=0.718505 avg=0.843603 VRAM=38.7GiB | 24.7% done | ETA(epoch): 1077s + [Epoch 74][20/66] loss=0.750713 avg=0.847847 VRAM=38.6GiB | 24.8% done | ETA(epoch): 884s + [Epoch 74][30/66] loss=0.962389 avg=0.859890 VRAM=38.7GiB | 24.8% done | ETA(epoch): 713s + [Epoch 74][40/66] loss=0.823883 avg=0.855494 VRAM=38.6GiB | 24.9% done | ETA(epoch): 511s + [Epoch 74][50/66] loss=0.867808 avg=0.854826 VRAM=38.7GiB | 24.9% done | ETA(epoch): 313s + [Epoch 74][60/66] loss=0.791953 avg=0.858014 VRAM=38.6GiB | 25.0% done | ETA(epoch): 117s + Train loss: 0.853648 (1287.1s) ETA: 5273min + Val loss: 0.899947 [t_0.0-0.2=1.0774 t_0.2-0.4=1.0058 t_0.4-0.6=0.8571 t_0.6-0.8=0.7361 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 75/299 (25% done) --- + [Epoch 75][10/66] loss=0.859433 avg=0.842057 VRAM=38.7GiB | 25.1% done | ETA(epoch): 1077s + [Epoch 75][20/66] loss=0.843725 avg=0.861940 VRAM=38.6GiB | 25.1% done | ETA(epoch): 885s + [Epoch 75][30/66] loss=0.762932 avg=0.860525 VRAM=38.7GiB | 25.2% done | ETA(epoch): 692s + [Epoch 75][40/66] loss=0.851173 avg=0.855586 VRAM=38.6GiB | 25.2% done | ETA(epoch): 500s + [Epoch 75][50/66] loss=0.997419 avg=0.865161 VRAM=38.7GiB | 25.3% done | ETA(epoch): 307s + [Epoch 75][60/66] loss=0.947841 avg=0.861194 VRAM=38.6GiB | 25.3% done | ETA(epoch): 115s + Train loss: 0.859227 (1268.5s) ETA: 5248min + Val loss: 0.834811 [t_0.0-0.2=1.0469 t_0.2-0.4=1.0432 t_0.4-0.6=0.8614 t_0.6-0.8=0.7245 t_0.8-1.0=0.6584] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 76/299 (25% done) --- + [Epoch 76][10/66] loss=0.885076 avg=0.888568 VRAM=38.7GiB | 25.4% done | ETA(epoch): 1075s + [Epoch 76][20/66] loss=0.794617 avg=0.876345 VRAM=38.6GiB | 25.4% done | ETA(epoch): 898s + [Epoch 76][30/66] loss=0.779956 avg=0.871980 VRAM=38.7GiB | 25.5% done | ETA(epoch): 699s + [Epoch 76][40/66] loss=0.941668 avg=0.862116 VRAM=38.6GiB | 25.5% done | ETA(epoch): 504s + [Epoch 76][50/66] loss=0.793881 avg=0.862314 VRAM=38.7GiB | 25.6% done | ETA(epoch): 309s + [Epoch 76][60/66] loss=0.756490 avg=0.863846 VRAM=38.6GiB | 25.6% done | ETA(epoch): 116s + Train loss: 0.865863 (1274.6s) ETA: 5222min + Val loss: 0.889505 [t_0.0-0.2=1.0900 t_0.2-0.4=0.9935 t_0.4-0.6=0.8542 t_0.6-0.8=0.7389 t_0.8-1.0=0.6550] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 77/299 (26% done) --- + [Epoch 77][10/66] loss=0.814564 avg=0.877046 VRAM=38.7GiB | 25.7% done | ETA(epoch): 1076s + [Epoch 77][20/66] loss=0.866348 avg=0.854217 VRAM=38.6GiB | 25.8% done | ETA(epoch): 883s + [Epoch 77][30/66] loss=0.912243 avg=0.864038 VRAM=38.7GiB | 25.8% done | ETA(epoch): 691s + [Epoch 77][40/66] loss=0.930006 avg=0.865184 VRAM=38.6GiB | 25.9% done | ETA(epoch): 499s + [Epoch 77][50/66] loss=0.831677 avg=0.859814 VRAM=38.7GiB | 25.9% done | ETA(epoch): 307s + [Epoch 77][60/66] loss=0.846731 avg=0.864198 VRAM=38.6GiB | 26.0% done | ETA(epoch): 115s + Train loss: 0.866652 (1269.5s) ETA: 5197min + Val loss: 0.884841 [t_0.0-0.2=1.0839 t_0.2-0.4=0.9942 t_0.4-0.6=0.8644 t_0.6-0.8=0.7069 t_0.8-1.0=0.6664] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 78/299 (26% done) --- + [Epoch 78][10/66] loss=0.710875 avg=0.846148 VRAM=38.7GiB | 26.1% done | ETA(epoch): 1076s + [Epoch 78][20/66] loss=0.831635 avg=0.835495 VRAM=38.6GiB | 26.1% done | ETA(epoch): 885s + [Epoch 78][30/66] loss=0.884655 avg=0.844981 VRAM=38.7GiB | 26.2% done | ETA(epoch): 693s + [Epoch 78][40/66] loss=0.793023 avg=0.849417 VRAM=38.6GiB | 26.2% done | ETA(epoch): 500s + [Epoch 78][50/66] loss=0.857578 avg=0.853295 VRAM=38.7GiB | 26.3% done | ETA(epoch): 308s + [Epoch 78][60/66] loss=0.796980 avg=0.859461 VRAM=38.6GiB | 26.3% done | ETA(epoch): 116s + Train loss: 0.860248 (1276.1s) ETA: 5172min + Val loss: 0.857455 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0064 t_0.4-0.6=0.8433 t_0.6-0.8=0.7338 t_0.8-1.0=0.6744] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 79/299 (26% done) --- + [Epoch 79][10/66] loss=0.786471 avg=0.897047 VRAM=38.7GiB | 26.4% done | ETA(epoch): 1076s + [Epoch 79][20/66] loss=0.842073 avg=0.883677 VRAM=38.6GiB | 26.4% done | ETA(epoch): 884s + [Epoch 79][30/66] loss=0.846776 avg=0.864862 VRAM=38.7GiB | 26.5% done | ETA(epoch): 692s + [Epoch 79][40/66] loss=0.796454 avg=0.853616 VRAM=38.6GiB | 26.5% done | ETA(epoch): 500s + [Epoch 79][50/66] loss=0.777696 avg=0.847084 VRAM=38.7GiB | 26.6% done | ETA(epoch): 308s + [Epoch 79][60/66] loss=0.915121 avg=0.850100 VRAM=38.6GiB | 26.6% done | ETA(epoch): 115s + Train loss: 0.853622 (1270.2s) ETA: 5147min + Val loss: 0.872057 [t_0.0-0.2=1.0580 t_0.2-0.4=0.9958 t_0.4-0.6=0.8595 t_0.6-0.8=0.7275 t_0.8-1.0=0.6574] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 80/299 (27% done) --- + [Epoch 80][10/66] loss=0.894971 avg=0.907434 VRAM=38.7GiB | 26.7% done | ETA(epoch): 1074s + [Epoch 80][20/66] loss=0.903653 avg=0.881259 VRAM=38.6GiB | 26.8% done | ETA(epoch): 882s + [Epoch 80][30/66] loss=0.921066 avg=0.867031 VRAM=38.7GiB | 26.8% done | ETA(epoch): 691s + [Epoch 80][40/66] loss=0.841665 avg=0.847211 VRAM=38.6GiB | 26.9% done | ETA(epoch): 499s + [Epoch 80][50/66] loss=0.778411 avg=0.860267 VRAM=38.7GiB | 26.9% done | ETA(epoch): 307s + [Epoch 80][60/66] loss=0.773638 avg=0.856862 VRAM=38.6GiB | 27.0% done | ETA(epoch): 115s + Train loss: 0.858388 (1267.8s) ETA: 5122min + Val loss: 0.838243 [t_0.0-0.2=1.0694 t_0.2-0.4=1.0163 t_0.4-0.6=0.8536 t_0.6-0.8=0.7291 t_0.8-1.0=0.6598] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 81/299 (27% done) --- + [Epoch 81][10/66] loss=0.873234 avg=0.862010 VRAM=38.7GiB | 27.1% done | ETA(epoch): 1075s + [Epoch 81][20/66] loss=0.829197 avg=0.857560 VRAM=38.6GiB | 27.1% done | ETA(epoch): 883s + [Epoch 81][30/66] loss=0.749396 avg=0.840661 VRAM=38.7GiB | 27.2% done | ETA(epoch): 691s + [Epoch 81][40/66] loss=0.943186 avg=0.858032 VRAM=38.6GiB | 27.2% done | ETA(epoch): 499s + [Epoch 81][50/66] loss=0.897852 avg=0.864724 VRAM=38.7GiB | 27.3% done | ETA(epoch): 307s + [Epoch 81][60/66] loss=0.823745 avg=0.863882 VRAM=38.6GiB | 27.3% done | ETA(epoch): 115s + Train loss: 0.861922 (1267.4s) ETA: 5096min + Val loss: 0.887448 [t_0.0-0.2=1.0809 t_0.2-0.4=0.9819 t_0.4-0.6=0.8540 t_0.6-0.8=0.7278 t_0.8-1.0=0.6495] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 82/299 (27% done) --- + [Epoch 82][10/66] loss=0.903860 avg=0.862976 VRAM=38.7GiB | 27.4% done | ETA(epoch): 1074s + [Epoch 82][20/66] loss=0.868579 avg=0.861188 VRAM=38.6GiB | 27.4% done | ETA(epoch): 883s + [Epoch 82][30/66] loss=0.774085 avg=0.848553 VRAM=38.7GiB | 27.5% done | ETA(epoch): 691s + [Epoch 82][40/66] loss=0.885742 avg=0.852228 VRAM=38.6GiB | 27.5% done | ETA(epoch): 499s + [Epoch 82][50/66] loss=0.911791 avg=0.856258 VRAM=38.7GiB | 27.6% done | ETA(epoch): 307s + [Epoch 82][60/66] loss=0.878034 avg=0.851299 VRAM=38.6GiB | 27.6% done | ETA(epoch): 115s + Train loss: 0.848567 (1267.7s) ETA: 5071min + Val loss: 0.873979 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0074 t_0.4-0.6=0.8484 t_0.6-0.8=0.7226 t_0.8-1.0=0.6773] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 83/299 (28% done) --- + [Epoch 83][10/66] loss=0.768116 avg=0.844077 VRAM=38.7GiB | 27.7% done | ETA(epoch): 1074s + [Epoch 83][20/66] loss=0.803401 avg=0.857848 VRAM=38.6GiB | 27.8% done | ETA(epoch): 887s + [Epoch 83][30/66] loss=0.945723 avg=0.871688 VRAM=38.7GiB | 27.8% done | ETA(epoch): 693s + [Epoch 83][40/66] loss=0.862625 avg=0.866637 VRAM=38.6GiB | 27.9% done | ETA(epoch): 500s + [Epoch 83][50/66] loss=0.782844 avg=0.867841 VRAM=38.7GiB | 27.9% done | ETA(epoch): 308s + [Epoch 83][60/66] loss=0.705502 avg=0.867656 VRAM=38.6GiB | 28.0% done | ETA(epoch): 115s + Train loss: 0.870977 (1269.2s) ETA: 5046min + Val loss: 0.841910 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0064 t_0.4-0.6=0.8446 t_0.6-0.8=0.7314 t_0.8-1.0=0.6721] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 84/299 (28% done) --- + [Epoch 84][10/66] loss=0.727832 avg=0.811222 VRAM=38.7GiB | 28.1% done | ETA(epoch): 1084s + [Epoch 84][20/66] loss=0.896837 avg=0.831837 VRAM=38.6GiB | 28.1% done | ETA(epoch): 887s + [Epoch 84][30/66] loss=0.859323 avg=0.838342 VRAM=38.7GiB | 28.2% done | ETA(epoch): 693s + [Epoch 84][40/66] loss=0.839562 avg=0.843073 VRAM=38.6GiB | 28.2% done | ETA(epoch): 501s + [Epoch 84][50/66] loss=0.968622 avg=0.846769 VRAM=38.7GiB | 28.3% done | ETA(epoch): 308s + [Epoch 84][60/66] loss=0.982684 avg=0.852260 VRAM=38.6GiB | 28.3% done | ETA(epoch): 115s + Train loss: 0.851944 (1269.4s) ETA: 5021min + Val loss: 0.838245 [t_0.0-0.2=1.0858 t_0.2-0.4=1.0058 t_0.4-0.6=0.8366 t_0.6-0.8=0.7011 t_0.8-1.0=0.6574] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 85/299 (28% done) --- + [Epoch 85][10/66] loss=0.894185 avg=0.896208 VRAM=38.7GiB | 28.4% done | ETA(epoch): 1074s + [Epoch 85][20/66] loss=0.807206 avg=0.875948 VRAM=38.6GiB | 28.4% done | ETA(epoch): 884s + [Epoch 85][30/66] loss=0.835396 avg=0.868214 VRAM=38.7GiB | 28.5% done | ETA(epoch): 691s + [Epoch 85][40/66] loss=0.760288 avg=0.865697 VRAM=38.6GiB | 28.5% done | ETA(epoch): 499s + [Epoch 85][50/66] loss=0.871349 avg=0.863910 VRAM=38.7GiB | 28.6% done | ETA(epoch): 307s + [Epoch 85][60/66] loss=0.826326 avg=0.859822 VRAM=38.6GiB | 28.6% done | ETA(epoch): 115s + Train loss: 0.858241 (1270.7s) ETA: 4996min + Val loss: 0.856389 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0004 t_0.4-0.6=0.8279 t_0.6-0.8=0.7024 t_0.8-1.0=0.6639] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 86/299 (29% done) --- + [Epoch 86][10/66] loss=0.754760 avg=0.858773 VRAM=38.7GiB | 28.7% done | ETA(epoch): 1074s + [Epoch 86][20/66] loss=0.940001 avg=0.860020 VRAM=38.6GiB | 28.8% done | ETA(epoch): 883s + [Epoch 86][30/66] loss=0.957542 avg=0.870207 VRAM=38.7GiB | 28.8% done | ETA(epoch): 691s + [Epoch 86][40/66] loss=0.901875 avg=0.865632 VRAM=38.6GiB | 28.9% done | ETA(epoch): 499s + [Epoch 86][50/66] loss=0.880954 avg=0.867079 VRAM=38.7GiB | 28.9% done | ETA(epoch): 307s + [Epoch 86][60/66] loss=0.772496 avg=0.864340 VRAM=38.6GiB | 29.0% done | ETA(epoch): 115s + Train loss: 0.860583 (1267.1s) ETA: 4971min + Val loss: 0.873477 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9778 t_0.4-0.6=0.8203 t_0.6-0.8=0.7249 t_0.8-1.0=0.6568] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 87/299 (29% done) --- + [Epoch 87][10/66] loss=0.897769 avg=0.883379 VRAM=38.7GiB | 29.1% done | ETA(epoch): 1073s + [Epoch 87][20/66] loss=0.751162 avg=0.876371 VRAM=38.6GiB | 29.1% done | ETA(epoch): 882s + [Epoch 87][30/66] loss=0.933010 avg=0.880757 VRAM=38.7GiB | 29.2% done | ETA(epoch): 690s + [Epoch 87][40/66] loss=0.799140 avg=0.876650 VRAM=38.6GiB | 29.2% done | ETA(epoch): 499s + [Epoch 87][50/66] loss=0.771625 avg=0.872668 VRAM=38.7GiB | 29.3% done | ETA(epoch): 307s + [Epoch 87][60/66] loss=0.773958 avg=0.868262 VRAM=38.6GiB | 29.3% done | ETA(epoch): 115s + Train loss: 0.864805 (1267.1s) ETA: 4947min + Val loss: 0.843520 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0035 t_0.4-0.6=0.8366 t_0.6-0.8=0.7354 t_0.8-1.0=0.6623] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 88/299 (29% done) --- + [Epoch 88][10/66] loss=0.768809 avg=0.897260 VRAM=38.7GiB | 29.4% done | ETA(epoch): 1075s + [Epoch 88][20/66] loss=0.877831 avg=0.897041 VRAM=38.6GiB | 29.4% done | ETA(epoch): 883s + [Epoch 88][30/66] loss=0.899738 avg=0.894098 VRAM=38.7GiB | 29.5% done | ETA(epoch): 691s + [Epoch 88][40/66] loss=0.714545 avg=0.890255 VRAM=38.6GiB | 29.5% done | ETA(epoch): 499s + [Epoch 88][50/66] loss=0.849110 avg=0.889926 VRAM=38.7GiB | 29.6% done | ETA(epoch): 307s + [Epoch 88][60/66] loss=0.940232 avg=0.883673 VRAM=38.6GiB | 29.6% done | ETA(epoch): 115s + Train loss: 0.880405 (1267.1s) ETA: 4922min + Val loss: 0.833310 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9613 t_0.4-0.6=0.8686 t_0.6-0.8=0.7304 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 89/299 (30% done) --- + [Epoch 89][10/66] loss=0.822786 avg=0.894006 VRAM=38.7GiB | 29.7% done | ETA(epoch): 1075s + [Epoch 89][20/66] loss=0.962482 avg=0.896169 VRAM=38.6GiB | 29.8% done | ETA(epoch): 883s + [Epoch 89][30/66] loss=0.911320 avg=0.894924 VRAM=38.7GiB | 29.8% done | ETA(epoch): 691s + [Epoch 89][40/66] loss=0.815756 avg=0.885896 VRAM=38.6GiB | 29.9% done | ETA(epoch): 499s + [Epoch 89][50/66] loss=0.891877 avg=0.880757 VRAM=38.7GiB | 29.9% done | ETA(epoch): 307s + [Epoch 89][60/66] loss=0.858401 avg=0.876955 VRAM=38.6GiB | 30.0% done | ETA(epoch): 115s + Train loss: 0.871604 (1268.1s) ETA: 4897min + Val loss: 0.874427 [t_0.0-0.2=1.0889 t_0.2-0.4=0.9860 t_0.4-0.6=0.8590 t_0.6-0.8=0.7259 t_0.8-1.0=0.6631] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 90/299 (30% done) --- + [MilestoneVis] train_0 step 5940 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 5940 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 5940 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 5940 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 5940 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 5940 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 5940 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 5940 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 5940 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 5940 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 90][10/66] loss=0.759873 avg=0.834812 VRAM=38.7GiB | 30.1% done | ETA(epoch): 6943s + [Epoch 90][20/66] loss=1.018129 avg=0.860174 VRAM=38.6GiB | 30.1% done | ETA(epoch): 3293s + [Epoch 90][30/66] loss=0.859460 avg=0.859537 VRAM=38.7GiB | 30.2% done | ETA(epoch): 1949s + [Epoch 90][40/66] loss=0.826952 avg=0.859808 VRAM=38.6GiB | 30.2% done | ETA(epoch): 1181s + [Epoch 90][50/66] loss=0.894607 avg=0.857245 VRAM=38.7GiB | 30.3% done | ETA(epoch): 643s + [Epoch 90][60/66] loss=0.933093 avg=0.860184 VRAM=38.6GiB | 30.3% done | ETA(epoch): 220s + Train loss: 0.856035 (2315.6s) ETA: 4912min + Val loss: 0.848417 [t_0.0-0.2=1.0794 t_0.2-0.4=1.0067 t_0.4-0.6=0.8774 t_0.6-0.8=0.7273 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 91/299 (30% done) --- + [Epoch 91][10/66] loss=0.737765 avg=0.853549 VRAM=38.7GiB | 30.4% done | ETA(epoch): 1073s + [Epoch 91][20/66] loss=0.827524 avg=0.857564 VRAM=38.6GiB | 30.4% done | ETA(epoch): 884s + [Epoch 91][30/66] loss=0.895859 avg=0.849995 VRAM=38.7GiB | 30.5% done | ETA(epoch): 692s + [Epoch 91][40/66] loss=0.738734 avg=0.845020 VRAM=38.6GiB | 30.5% done | ETA(epoch): 499s + [Epoch 91][50/66] loss=0.822293 avg=0.845822 VRAM=38.7GiB | 30.6% done | ETA(epoch): 307s + [Epoch 91][60/66] loss=1.031441 avg=0.846514 VRAM=38.6GiB | 30.6% done | ETA(epoch): 115s + Train loss: 0.850543 (1267.5s) ETA: 4887min + Val loss: 0.871515 [t_0.0-0.2=1.0796 t_0.2-0.4=0.9921 t_0.4-0.6=0.8758 t_0.6-0.8=0.6983 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 92/299 (31% done) --- + [Epoch 92][10/66] loss=0.806779 avg=0.889519 VRAM=38.7GiB | 30.7% done | ETA(epoch): 1073s + [Epoch 92][20/66] loss=0.890523 avg=0.863537 VRAM=38.6GiB | 30.8% done | ETA(epoch): 883s + [Epoch 92][30/66] loss=0.969335 avg=0.875155 VRAM=38.7GiB | 30.8% done | ETA(epoch): 691s + [Epoch 92][40/66] loss=0.940190 avg=0.868956 VRAM=38.6GiB | 30.9% done | ETA(epoch): 499s + [Epoch 92][50/66] loss=0.844275 avg=0.868155 VRAM=38.7GiB | 30.9% done | ETA(epoch): 307s + [Epoch 92][60/66] loss=0.785355 avg=0.870127 VRAM=38.6GiB | 31.0% done | ETA(epoch): 115s + Train loss: 0.869093 (1266.9s) ETA: 4862min + Val loss: 0.847903 [t_0.0-0.2=1.0851 t_0.2-0.4=0.9484 t_0.4-0.6=0.8440 t_0.6-0.8=0.6971 t_0.8-1.0=0.6523] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 93/299 (31% done) --- + [Epoch 93][10/66] loss=0.789242 avg=0.824367 VRAM=38.7GiB | 31.1% done | ETA(epoch): 1091s + [Epoch 93][20/66] loss=0.816142 avg=0.831229 VRAM=38.6GiB | 31.1% done | ETA(epoch): 890s + [Epoch 93][30/66] loss=0.879848 avg=0.840729 VRAM=38.7GiB | 31.2% done | ETA(epoch): 695s + [Epoch 93][40/66] loss=0.852486 avg=0.856810 VRAM=38.6GiB | 31.2% done | ETA(epoch): 501s + [Epoch 93][50/66] loss=0.791593 avg=0.856719 VRAM=38.7GiB | 31.3% done | ETA(epoch): 308s + [Epoch 93][60/66] loss=0.852023 avg=0.863209 VRAM=38.6GiB | 31.3% done | ETA(epoch): 116s + Train loss: 0.868233 (1270.4s) ETA: 4837min + Val loss: 0.851137 [t_0.0-0.2=1.0854 t_0.2-0.4=0.9769 t_0.4-0.6=0.8557 t_0.6-0.8=0.7228 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 19.8/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 94/299 (31% done) --- + [Epoch 94][10/66] loss=0.804330 avg=0.830798 VRAM=38.7GiB | 31.4% done | ETA(epoch): 1074s + [Epoch 94][20/66] loss=0.903578 avg=0.859064 VRAM=38.6GiB | 31.4% done | ETA(epoch): 882s + [Epoch 94][30/66] loss=0.796290 avg=0.860333 VRAM=38.7GiB | 31.5% done | ETA(epoch): 691s + [Epoch 94][40/66] loss=0.786012 avg=0.860699 VRAM=38.6GiB | 31.5% done | ETA(epoch): 499s + [Epoch 94][50/66] loss=0.808764 avg=0.861938 VRAM=38.7GiB | 31.6% done | ETA(epoch): 307s + [Epoch 94][60/66] loss=0.921456 avg=0.862108 VRAM=38.6GiB | 31.6% done | ETA(epoch): 115s + Train loss: 0.862396 (1268.3s) ETA: 4812min + Val loss: 0.842584 [t_0.0-0.2=1.0656 t_0.2-0.4=0.9975 t_0.4-0.6=0.8399 t_0.6-0.8=0.7268 t_0.8-1.0=0.6434] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 95/299 (32% done) --- + [Epoch 95][10/66] loss=0.829399 avg=0.832191 VRAM=38.7GiB | 31.7% done | ETA(epoch): 1074s + [Epoch 95][20/66] loss=0.936516 avg=0.856105 VRAM=38.6GiB | 31.8% done | ETA(epoch): 883s + [Epoch 95][30/66] loss=0.899032 avg=0.859344 VRAM=38.7GiB | 31.8% done | ETA(epoch): 691s + [Epoch 95][40/66] loss=0.872707 avg=0.854203 VRAM=38.6GiB | 31.9% done | ETA(epoch): 499s + [Epoch 95][50/66] loss=0.833289 avg=0.862313 VRAM=38.7GiB | 31.9% done | ETA(epoch): 307s + [Epoch 95][60/66] loss=0.846850 avg=0.864653 VRAM=38.6GiB | 32.0% done | ETA(epoch): 115s + Train loss: 0.864475 (1269.7s) ETA: 4787min + Val loss: 0.813897 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9463 t_0.4-0.6=0.8382 t_0.6-0.8=0.7115 t_0.8-1.0=0.6493] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0095 (BEST) + Deleted old checkpoint: checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 96/299 (32% done) --- + [Epoch 96][10/66] loss=0.781842 avg=0.846492 VRAM=38.7GiB | 32.1% done | ETA(epoch): 1077s + [Epoch 96][20/66] loss=0.801173 avg=0.860613 VRAM=38.6GiB | 32.1% done | ETA(epoch): 884s + [Epoch 96][30/66] loss=0.918636 avg=0.864435 VRAM=38.7GiB | 32.2% done | ETA(epoch): 691s + [Epoch 96][40/66] loss=0.849381 avg=0.854338 VRAM=38.6GiB | 32.2% done | ETA(epoch): 499s + [Epoch 96][50/66] loss=0.869656 avg=0.859899 VRAM=38.7GiB | 32.3% done | ETA(epoch): 313s + [Epoch 96][60/66] loss=0.911374 avg=0.861963 VRAM=38.6GiB | 32.3% done | ETA(epoch): 117s + Train loss: 0.863516 (1287.1s) ETA: 4762min + Val loss: 0.842300 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9889 t_0.4-0.6=0.8584 t_0.6-0.8=0.7164 t_0.8-1.0=0.6650] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 19.5/188.5 GiB (10.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 97/299 (32% done) --- + [Epoch 97][10/66] loss=0.937833 avg=0.887353 VRAM=38.7GiB | 32.4% done | ETA(epoch): 1075s + [Epoch 97][20/66] loss=0.860866 avg=0.873778 VRAM=38.6GiB | 32.4% done | ETA(epoch): 883s + [Epoch 97][30/66] loss=0.828533 avg=0.877767 VRAM=38.7GiB | 32.5% done | ETA(epoch): 691s + [Epoch 97][40/66] loss=0.942115 avg=0.865536 VRAM=38.6GiB | 32.5% done | ETA(epoch): 499s + [Epoch 97][50/66] loss=0.941356 avg=0.866024 VRAM=38.7GiB | 32.6% done | ETA(epoch): 307s + [Epoch 97][60/66] loss=0.837143 avg=0.860793 VRAM=38.6GiB | 32.6% done | ETA(epoch): 115s + Train loss: 0.864756 (1267.5s) ETA: 4737min + Val loss: 0.852580 [t_0.0-0.2=1.0766 t_0.2-0.4=1.0272 t_0.4-0.6=0.8612 t_0.6-0.8=0.7073 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 98/299 (33% done) --- + [Epoch 98][10/66] loss=0.987492 avg=0.849812 VRAM=38.7GiB | 32.7% done | ETA(epoch): 1076s + [Epoch 98][20/66] loss=0.766366 avg=0.849266 VRAM=38.6GiB | 32.8% done | ETA(epoch): 884s + [Epoch 98][30/66] loss=0.989442 avg=0.855450 VRAM=38.7GiB | 32.8% done | ETA(epoch): 694s + [Epoch 98][40/66] loss=0.851608 avg=0.861278 VRAM=38.6GiB | 32.9% done | ETA(epoch): 501s + [Epoch 98][50/66] loss=0.807638 avg=0.858684 VRAM=38.7GiB | 32.9% done | ETA(epoch): 308s + [Epoch 98][60/66] loss=0.867773 avg=0.857908 VRAM=38.6GiB | 33.0% done | ETA(epoch): 115s + Train loss: 0.861672 (1270.0s) ETA: 4712min + Val loss: 0.844169 [t_0.0-0.2=1.0637 t_0.2-0.4=0.9745 t_0.4-0.6=0.8770 t_0.6-0.8=0.7276 t_0.8-1.0=0.6594] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0098 +[MEM @ epoch 98 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 99/299 (33% done) --- + [Epoch 99][10/66] loss=0.679792 avg=0.865722 VRAM=38.7GiB | 33.1% done | ETA(epoch): 1075s + [Epoch 99][20/66] loss=0.749062 avg=0.861549 VRAM=38.6GiB | 33.1% done | ETA(epoch): 883s + [Epoch 99][30/66] loss=0.835393 avg=0.860883 VRAM=38.7GiB | 33.2% done | ETA(epoch): 691s + [Epoch 99][40/66] loss=0.850963 avg=0.853638 VRAM=38.6GiB | 33.2% done | ETA(epoch): 499s + [Epoch 99][50/66] loss=0.830811 avg=0.856753 VRAM=38.7GiB | 33.3% done | ETA(epoch): 307s + [Epoch 99][60/66] loss=0.783823 avg=0.850992 VRAM=38.6GiB | 33.3% done | ETA(epoch): 115s + Train loss: 0.848351 (1267.2s) ETA: 4687min + Val loss: 0.881852 [t_0.0-0.2=1.0882 t_0.2-0.4=1.0172 t_0.4-0.6=0.8131 t_0.6-0.8=0.7268 t_0.8-1.0=0.6552] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 100/299 (33% done) --- + [Epoch 100][10/66] loss=0.707337 avg=0.826334 VRAM=38.7GiB | 33.4% done | ETA(epoch): 1093s + [Epoch 100][20/66] loss=0.862292 avg=0.852078 VRAM=38.6GiB | 33.4% done | ETA(epoch): 891s + [Epoch 100][30/66] loss=0.849977 avg=0.857029 VRAM=38.7GiB | 33.5% done | ETA(epoch): 695s + [Epoch 100][40/66] loss=0.758742 avg=0.868305 VRAM=38.6GiB | 33.5% done | ETA(epoch): 501s + [Epoch 100][50/66] loss=0.937653 avg=0.872226 VRAM=38.7GiB | 33.6% done | ETA(epoch): 308s + [Epoch 100][60/66] loss=1.010351 avg=0.876112 VRAM=38.6GiB | 33.6% done | ETA(epoch): 116s + Train loss: 0.876511 (1270.6s) ETA: 4663min + Val loss: 0.841915 [t_0.0-0.2=1.0608 t_0.2-0.4=1.0381 t_0.4-0.6=0.8698 t_0.6-0.8=0.7129 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 101/299 (34% done) --- + [Epoch 101][10/66] loss=0.798365 avg=0.862447 VRAM=38.7GiB | 33.7% done | ETA(epoch): 1076s + [Epoch 101][20/66] loss=0.764806 avg=0.860402 VRAM=38.6GiB | 33.8% done | ETA(epoch): 883s + [Epoch 101][30/66] loss=0.932255 avg=0.870261 VRAM=38.7GiB | 33.8% done | ETA(epoch): 691s + [Epoch 101][40/66] loss=0.901447 avg=0.865949 VRAM=38.6GiB | 33.9% done | ETA(epoch): 499s + [Epoch 101][50/66] loss=0.899411 avg=0.862845 VRAM=38.7GiB | 33.9% done | ETA(epoch): 307s + [Epoch 101][60/66] loss=0.851224 avg=0.855968 VRAM=38.6GiB | 34.0% done | ETA(epoch): 115s + Train loss: 0.857470 (1267.2s) ETA: 4638min + Val loss: 0.859045 [t_0.0-0.2=1.0919 t_0.2-0.4=0.9764 t_0.4-0.6=0.8484 t_0.6-0.8=0.7311 t_0.8-1.0=0.6531] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 102/299 (34% done) --- + [Epoch 102][10/66] loss=0.819327 avg=0.858897 VRAM=38.7GiB | 34.1% done | ETA(epoch): 1076s + [Epoch 102][20/66] loss=0.890353 avg=0.839417 VRAM=38.6GiB | 34.1% done | ETA(epoch): 883s + [Epoch 102][30/66] loss=0.852891 avg=0.844558 VRAM=38.7GiB | 34.2% done | ETA(epoch): 691s + [Epoch 102][40/66] loss=0.998418 avg=0.851381 VRAM=38.6GiB | 34.2% done | ETA(epoch): 499s + [Epoch 102][50/66] loss=0.906683 avg=0.855532 VRAM=38.7GiB | 34.3% done | ETA(epoch): 307s + [Epoch 102][60/66] loss=0.831913 avg=0.862141 VRAM=38.6GiB | 34.3% done | ETA(epoch): 115s + Train loss: 0.863573 (1267.3s) ETA: 4613min + Val loss: 0.880758 [t_0.0-0.2=1.0725 t_0.2-0.4=1.0106 t_0.4-0.6=0.8408 t_0.6-0.8=0.7128 t_0.8-1.0=0.6817] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 103/299 (34% done) --- + [Epoch 103][10/66] loss=0.768569 avg=0.888777 VRAM=38.7GiB | 34.4% done | ETA(epoch): 1075s + [Epoch 103][20/66] loss=0.776707 avg=0.862823 VRAM=38.6GiB | 34.4% done | ETA(epoch): 883s + [Epoch 103][30/66] loss=0.860144 avg=0.869215 VRAM=38.7GiB | 34.5% done | ETA(epoch): 691s + [Epoch 103][40/66] loss=0.872193 avg=0.863494 VRAM=38.6GiB | 34.5% done | ETA(epoch): 499s + [Epoch 103][50/66] loss=0.781946 avg=0.858345 VRAM=38.7GiB | 34.6% done | ETA(epoch): 307s + [Epoch 103][60/66] loss=0.926681 avg=0.858074 VRAM=38.6GiB | 34.6% done | ETA(epoch): 115s + Train loss: 0.855992 (1267.1s) ETA: 4588min + Val loss: 0.841009 [t_0.0-0.2=1.0758 t_0.2-0.4=1.0272 t_0.4-0.6=0.8502 t_0.6-0.8=0.6756 t_0.8-1.0=0.6527] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 104/299 (35% done) --- + [Epoch 104][10/66] loss=0.855486 avg=0.851349 VRAM=38.7GiB | 34.7% done | ETA(epoch): 1076s + [Epoch 104][20/66] loss=0.793957 avg=0.845954 VRAM=38.6GiB | 34.8% done | ETA(epoch): 882s + [Epoch 104][30/66] loss=0.912438 avg=0.845964 VRAM=38.7GiB | 34.8% done | ETA(epoch): 691s + [Epoch 104][40/66] loss=0.754521 avg=0.865309 VRAM=38.6GiB | 34.9% done | ETA(epoch): 500s + [Epoch 104][50/66] loss=0.872342 avg=0.860091 VRAM=38.7GiB | 34.9% done | ETA(epoch): 308s + [Epoch 104][60/66] loss=0.742773 avg=0.860288 VRAM=38.6GiB | 35.0% done | ETA(epoch): 115s + Train loss: 0.857122 (1268.8s) ETA: 4564min + Val loss: 0.811569 [t_0.0-0.2=1.0744 t_0.2-0.4=0.9887 t_0.4-0.6=0.8204 t_0.6-0.8=0.7182 t_0.8-1.0=0.6501] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0104 (BEST) + Deleted old checkpoint: checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 105/299 (35% done) --- + [Epoch 105][10/66] loss=0.842854 avg=0.875085 VRAM=38.7GiB | 35.1% done | ETA(epoch): 1074s + [Epoch 105][20/66] loss=1.061811 avg=0.881139 VRAM=38.6GiB | 35.1% done | ETA(epoch): 882s + [Epoch 105][30/66] loss=0.828484 avg=0.865721 VRAM=38.7GiB | 35.2% done | ETA(epoch): 690s + [Epoch 105][40/66] loss=1.016454 avg=0.877483 VRAM=38.6GiB | 35.2% done | ETA(epoch): 510s + [Epoch 105][50/66] loss=0.931515 avg=0.868579 VRAM=38.7GiB | 35.3% done | ETA(epoch): 313s + [Epoch 105][60/66] loss=0.791349 avg=0.865732 VRAM=38.6GiB | 35.3% done | ETA(epoch): 117s + Train loss: 0.867332 (1284.6s) ETA: 4539min + Val loss: 0.842561 [t_0.0-0.2=1.0842 t_0.2-0.4=0.9826 t_0.4-0.6=0.8516 t_0.6-0.8=0.6968 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 106/299 (35% done) --- + [Epoch 106][10/66] loss=0.822894 avg=0.841237 VRAM=38.7GiB | 35.4% done | ETA(epoch): 1074s + [Epoch 106][20/66] loss=0.889432 avg=0.863572 VRAM=38.6GiB | 35.4% done | ETA(epoch): 883s + [Epoch 106][30/66] loss=0.915390 avg=0.864090 VRAM=38.7GiB | 35.5% done | ETA(epoch): 691s + [Epoch 106][40/66] loss=0.815298 avg=0.859378 VRAM=38.6GiB | 35.5% done | ETA(epoch): 499s + [Epoch 106][50/66] loss=0.931338 avg=0.856359 VRAM=38.7GiB | 35.6% done | ETA(epoch): 307s + [Epoch 106][60/66] loss=0.977590 avg=0.867957 VRAM=38.6GiB | 35.6% done | ETA(epoch): 115s + Train loss: 0.868391 (1267.3s) ETA: 4515min + Val loss: 0.859613 [t_0.0-0.2=1.0619 t_0.2-0.4=1.0085 t_0.4-0.6=0.8219 t_0.6-0.8=0.7306 t_0.8-1.0=0.6342] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 107/299 (36% done) --- + [Epoch 107][10/66] loss=1.023521 avg=0.876832 VRAM=38.7GiB | 35.7% done | ETA(epoch): 1075s + [Epoch 107][20/66] loss=0.784307 avg=0.857386 VRAM=38.6GiB | 35.8% done | ETA(epoch): 892s + [Epoch 107][30/66] loss=0.815456 avg=0.850512 VRAM=38.7GiB | 35.8% done | ETA(epoch): 696s + [Epoch 107][40/66] loss=0.769345 avg=0.862148 VRAM=38.6GiB | 35.9% done | ETA(epoch): 501s + [Epoch 107][50/66] loss=0.978483 avg=0.863074 VRAM=38.7GiB | 35.9% done | ETA(epoch): 308s + [Epoch 107][60/66] loss=0.892956 avg=0.863468 VRAM=38.6GiB | 36.0% done | ETA(epoch): 116s + Train loss: 0.865041 (1270.8s) ETA: 4490min + Val loss: 0.883903 [t_0.0-0.2=1.0744 t_0.2-0.4=1.0134 t_0.4-0.6=0.8598 t_0.6-0.8=0.7127 t_0.8-1.0=0.6429] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0107 +[MEM @ epoch 107 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 108/299 (36% done) --- + [Epoch 108][10/66] loss=0.918885 avg=0.873589 VRAM=38.7GiB | 36.1% done | ETA(epoch): 1087s + [Epoch 108][20/66] loss=0.971575 avg=0.862435 VRAM=38.6GiB | 36.1% done | ETA(epoch): 888s + [Epoch 108][30/66] loss=0.756556 avg=0.853152 VRAM=38.7GiB | 36.2% done | ETA(epoch): 694s + [Epoch 108][40/66] loss=0.955186 avg=0.861690 VRAM=38.6GiB | 36.2% done | ETA(epoch): 501s + [Epoch 108][50/66] loss=0.797275 avg=0.861806 VRAM=38.7GiB | 36.3% done | ETA(epoch): 308s + [Epoch 108][60/66] loss=0.859753 avg=0.861895 VRAM=38.6GiB | 36.3% done | ETA(epoch): 116s + Train loss: 0.860716 (1270.5s) ETA: 4466min + Val loss: 0.872801 [t_0.0-0.2=1.0830 t_0.2-0.4=1.0071 t_0.4-0.6=0.8353 t_0.6-0.8=0.7086 t_0.8-1.0=0.6685] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 109/299 (36% done) --- + [Epoch 109][10/66] loss=1.020916 avg=0.902935 VRAM=38.7GiB | 36.4% done | ETA(epoch): 1074s + [Epoch 109][20/66] loss=0.856909 avg=0.883469 VRAM=38.6GiB | 36.4% done | ETA(epoch): 883s + [Epoch 109][30/66] loss=0.802624 avg=0.875493 VRAM=38.7GiB | 36.5% done | ETA(epoch): 691s + [Epoch 109][40/66] loss=0.834306 avg=0.866127 VRAM=38.6GiB | 36.5% done | ETA(epoch): 499s + [Epoch 109][50/66] loss=0.896399 avg=0.868294 VRAM=38.7GiB | 36.6% done | ETA(epoch): 307s + [Epoch 109][60/66] loss=0.908138 avg=0.876225 VRAM=38.6GiB | 36.6% done | ETA(epoch): 115s + Train loss: 0.877800 (1272.3s) ETA: 4441min + Val loss: 0.861098 [t_0.0-0.2=1.0552 t_0.2-0.4=0.9945 t_0.4-0.6=0.8486 t_0.6-0.8=0.7160 t_0.8-1.0=0.6588] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 110/299 (37% done) --- + [Epoch 110][10/66] loss=0.813677 avg=0.836899 VRAM=38.7GiB | 36.7% done | ETA(epoch): 1074s + [Epoch 110][20/66] loss=0.848039 avg=0.817130 VRAM=38.6GiB | 36.8% done | ETA(epoch): 883s + [Epoch 110][30/66] loss=0.775279 avg=0.838072 VRAM=38.7GiB | 36.8% done | ETA(epoch): 691s + [Epoch 110][40/66] loss=0.952471 avg=0.845673 VRAM=38.6GiB | 36.9% done | ETA(epoch): 499s + [Epoch 110][50/66] loss=0.918807 avg=0.852464 VRAM=38.7GiB | 36.9% done | ETA(epoch): 307s + [Epoch 110][60/66] loss=0.776143 avg=0.850361 VRAM=38.6GiB | 37.0% done | ETA(epoch): 115s + Train loss: 0.852819 (1269.7s) ETA: 4417min + Val loss: 0.878836 [t_0.0-0.2=1.0742 t_0.2-0.4=1.0237 t_0.4-0.6=0.8422 t_0.6-0.8=0.7434 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 111/299 (37% done) --- + [Epoch 111][10/66] loss=0.769104 avg=0.867349 VRAM=38.7GiB | 37.1% done | ETA(epoch): 1075s + [Epoch 111][20/66] loss=0.859824 avg=0.881804 VRAM=38.6GiB | 37.1% done | ETA(epoch): 883s + [Epoch 111][30/66] loss=0.702154 avg=0.873571 VRAM=38.7GiB | 37.2% done | ETA(epoch): 691s + [Epoch 111][40/66] loss=0.895654 avg=0.869507 VRAM=38.6GiB | 37.2% done | ETA(epoch): 499s + [Epoch 111][50/66] loss=0.835729 avg=0.863139 VRAM=38.7GiB | 37.3% done | ETA(epoch): 307s + [Epoch 111][60/66] loss=0.909972 avg=0.857175 VRAM=38.6GiB | 37.3% done | ETA(epoch): 115s + Train loss: 0.858032 (1267.9s) ETA: 4392min + Val loss: 0.824633 [t_0.0-0.2=1.0931 t_0.2-0.4=1.0037 t_0.4-0.6=0.8272 t_0.6-0.8=0.7162 t_0.8-1.0=0.6469] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 112/299 (37% done) --- + [Epoch 112][10/66] loss=0.725716 avg=0.820207 VRAM=38.7GiB | 37.4% done | ETA(epoch): 1075s + [Epoch 112][20/66] loss=0.820025 avg=0.836291 VRAM=38.6GiB | 37.4% done | ETA(epoch): 884s + [Epoch 112][30/66] loss=0.800888 avg=0.853900 VRAM=38.7GiB | 37.5% done | ETA(epoch): 692s + [Epoch 112][40/66] loss=0.771399 avg=0.863454 VRAM=38.6GiB | 37.5% done | ETA(epoch): 501s + [Epoch 112][50/66] loss=0.836189 avg=0.866768 VRAM=38.7GiB | 37.6% done | ETA(epoch): 308s + [Epoch 112][60/66] loss=0.816546 avg=0.866800 VRAM=38.6GiB | 37.6% done | ETA(epoch): 115s + Train loss: 0.870582 (1269.9s) ETA: 4368min + Val loss: 0.852974 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0025 t_0.4-0.6=0.8487 t_0.6-0.8=0.6887 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 113/299 (38% done) --- + [Epoch 113][10/66] loss=0.918006 avg=0.865943 VRAM=38.7GiB | 37.7% done | ETA(epoch): 1074s + [Epoch 113][20/66] loss=0.813233 avg=0.859364 VRAM=38.6GiB | 37.8% done | ETA(epoch): 882s + [Epoch 113][30/66] loss=0.933331 avg=0.860306 VRAM=38.7GiB | 37.8% done | ETA(epoch): 691s + [Epoch 113][40/66] loss=0.834739 avg=0.867955 VRAM=38.6GiB | 37.9% done | ETA(epoch): 499s + [Epoch 113][50/66] loss=0.836336 avg=0.872393 VRAM=38.7GiB | 37.9% done | ETA(epoch): 307s + [Epoch 113][60/66] loss=0.839982 avg=0.861995 VRAM=38.6GiB | 38.0% done | ETA(epoch): 115s + Train loss: 0.862170 (1267.5s) ETA: 4344min + Val loss: 0.847482 [t_0.0-0.2=1.0785 t_0.2-0.4=0.9851 t_0.4-0.6=0.8657 t_0.6-0.8=0.7265 t_0.8-1.0=0.6659] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 114/299 (38% done) --- + [Epoch 114][10/66] loss=0.872971 avg=0.853259 VRAM=38.7GiB | 38.1% done | ETA(epoch): 1075s + [Epoch 114][20/66] loss=0.770475 avg=0.851473 VRAM=38.6GiB | 38.1% done | ETA(epoch): 883s + [Epoch 114][30/66] loss=0.904781 avg=0.856303 VRAM=38.7GiB | 38.2% done | ETA(epoch): 692s + [Epoch 114][40/66] loss=0.701344 avg=0.849344 VRAM=38.6GiB | 38.2% done | ETA(epoch): 499s + [Epoch 114][50/66] loss=0.794602 avg=0.850215 VRAM=38.7GiB | 38.3% done | ETA(epoch): 307s + [Epoch 114][60/66] loss=0.811940 avg=0.846805 VRAM=38.6GiB | 38.3% done | ETA(epoch): 115s + Train loss: 0.850057 (1268.2s) ETA: 4319min + Val loss: 0.891056 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0165 t_0.4-0.6=0.8172 t_0.6-0.8=0.7369 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 19.5/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 115/299 (38% done) --- + [Epoch 115][10/66] loss=0.948415 avg=0.873687 VRAM=38.7GiB | 38.4% done | ETA(epoch): 1077s + [Epoch 115][20/66] loss=0.775373 avg=0.847412 VRAM=38.6GiB | 38.4% done | ETA(epoch): 885s + [Epoch 115][30/66] loss=0.774699 avg=0.850346 VRAM=38.7GiB | 38.5% done | ETA(epoch): 693s + [Epoch 115][40/66] loss=0.905183 avg=0.854816 VRAM=38.6GiB | 38.5% done | ETA(epoch): 500s + [Epoch 115][50/66] loss=0.849527 avg=0.858814 VRAM=38.7GiB | 38.6% done | ETA(epoch): 308s + [Epoch 115][60/66] loss=0.819797 avg=0.853045 VRAM=38.6GiB | 38.6% done | ETA(epoch): 115s + Train loss: 0.851671 (1269.2s) ETA: 4295min + Val loss: 0.870576 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0050 t_0.4-0.6=0.8622 t_0.6-0.8=0.7048 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 116/299 (39% done) --- + [Epoch 116][10/66] loss=0.910492 avg=0.862083 VRAM=38.7GiB | 38.7% done | ETA(epoch): 1097s + [Epoch 116][20/66] loss=0.953223 avg=0.850827 VRAM=38.6GiB | 38.8% done | ETA(epoch): 892s + [Epoch 116][30/66] loss=0.905785 avg=0.847631 VRAM=38.7GiB | 38.8% done | ETA(epoch): 696s + [Epoch 116][40/66] loss=0.830655 avg=0.848074 VRAM=38.6GiB | 38.9% done | ETA(epoch): 502s + [Epoch 116][50/66] loss=0.814507 avg=0.845591 VRAM=38.7GiB | 38.9% done | ETA(epoch): 309s + [Epoch 116][60/66] loss=0.814340 avg=0.839113 VRAM=38.6GiB | 39.0% done | ETA(epoch): 116s + Train loss: 0.839892 (1271.8s) ETA: 4271min + Val loss: 0.848096 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9990 t_0.4-0.6=0.8221 t_0.6-0.8=0.7184 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 117/299 (39% done) --- + [Epoch 117][10/66] loss=0.991918 avg=0.898374 VRAM=38.7GiB | 39.1% done | ETA(epoch): 1075s + [Epoch 117][20/66] loss=0.897056 avg=0.893629 VRAM=38.6GiB | 39.1% done | ETA(epoch): 884s + [Epoch 117][30/66] loss=0.896946 avg=0.887703 VRAM=38.7GiB | 39.2% done | ETA(epoch): 692s + [Epoch 117][40/66] loss=0.817093 avg=0.882021 VRAM=38.6GiB | 39.2% done | ETA(epoch): 500s + [Epoch 117][50/66] loss=0.850016 avg=0.874241 VRAM=38.7GiB | 39.3% done | ETA(epoch): 308s + [Epoch 117][60/66] loss=0.845396 avg=0.870545 VRAM=38.6GiB | 39.3% done | ETA(epoch): 116s + Train loss: 0.868508 (1272.4s) ETA: 4247min + Val loss: 0.867382 [t_0.0-0.2=1.0701 t_0.2-0.4=0.9988 t_0.4-0.6=0.7830 t_0.6-0.8=0.7305 t_0.8-1.0=0.6518] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 118/299 (39% done) --- + [Epoch 118][10/66] loss=0.957646 avg=0.859526 VRAM=38.7GiB | 39.4% done | ETA(epoch): 1075s + [Epoch 118][20/66] loss=0.812602 avg=0.850654 VRAM=38.6GiB | 39.4% done | ETA(epoch): 883s + [Epoch 118][30/66] loss=0.721708 avg=0.835873 VRAM=38.7GiB | 39.5% done | ETA(epoch): 691s + [Epoch 118][40/66] loss=0.948969 avg=0.848118 VRAM=38.6GiB | 39.5% done | ETA(epoch): 499s + [Epoch 118][50/66] loss=0.835788 avg=0.844155 VRAM=38.7GiB | 39.6% done | ETA(epoch): 307s + [Epoch 118][60/66] loss=0.974827 avg=0.844750 VRAM=38.6GiB | 39.6% done | ETA(epoch): 115s + Train loss: 0.849348 (1268.1s) ETA: 4222min + Val loss: 0.871310 [t_0.0-0.2=1.0559 t_0.2-0.4=1.0172 t_0.4-0.6=0.8748 t_0.6-0.8=0.7037 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 19.6/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 119/299 (40% done) --- + [Epoch 119][10/66] loss=0.844312 avg=0.877483 VRAM=38.7GiB | 39.7% done | ETA(epoch): 1075s + [Epoch 119][20/66] loss=0.884663 avg=0.875649 VRAM=38.6GiB | 39.8% done | ETA(epoch): 883s + [Epoch 119][30/66] loss=0.886157 avg=0.868869 VRAM=38.7GiB | 39.8% done | ETA(epoch): 692s + [Epoch 119][40/66] loss=0.849521 avg=0.873225 VRAM=38.6GiB | 39.9% done | ETA(epoch): 499s + [Epoch 119][50/66] loss=0.845381 avg=0.869671 VRAM=38.7GiB | 39.9% done | ETA(epoch): 308s + [Epoch 119][60/66] loss=0.901835 avg=0.872608 VRAM=38.6GiB | 40.0% done | ETA(epoch): 115s + Train loss: 0.866268 (1269.8s) ETA: 4198min + Val loss: 0.847478 [t_0.0-0.2=1.0732 t_0.2-0.4=1.0020 t_0.4-0.6=0.8283 t_0.6-0.8=0.7155 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 120/299 (40% done) --- + [MilestoneVis] train_0 step 7920 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 7920 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 7920 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 7920 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 7920 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 7920 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 7920 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 7920 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 7920 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 7920 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 120][10/66] loss=0.972220 avg=0.855851 VRAM=38.7GiB | 40.1% done | ETA(epoch): 6849s + [Epoch 120][20/66] loss=0.842627 avg=0.852267 VRAM=38.6GiB | 40.1% done | ETA(epoch): 3254s + [Epoch 120][30/66] loss=0.908397 avg=0.851938 VRAM=38.7GiB | 40.2% done | ETA(epoch): 1928s + [Epoch 120][40/66] loss=0.994705 avg=0.848798 VRAM=38.6GiB | 40.2% done | ETA(epoch): 1169s + [Epoch 120][50/66] loss=0.875831 avg=0.851590 VRAM=38.7GiB | 40.3% done | ETA(epoch): 642s + [Epoch 120][60/66] loss=0.783125 avg=0.854045 VRAM=38.6GiB | 40.3% done | ETA(epoch): 220s + Train loss: 0.853371 (2314.6s) ETA: 4200min + Val loss: 0.852036 [t_0.0-0.2=1.0500 t_0.2-0.4=1.0198 t_0.4-0.6=0.8632 t_0.6-0.8=0.7150 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 121/299 (40% done) --- + [Epoch 121][10/66] loss=0.896852 avg=0.899315 VRAM=38.7GiB | 40.4% done | ETA(epoch): 1073s + [Epoch 121][20/66] loss=0.865983 avg=0.863900 VRAM=38.6GiB | 40.4% done | ETA(epoch): 882s + [Epoch 121][30/66] loss=0.926843 avg=0.866099 VRAM=38.7GiB | 40.5% done | ETA(epoch): 690s + [Epoch 121][40/66] loss=0.999985 avg=0.868595 VRAM=38.6GiB | 40.5% done | ETA(epoch): 499s + [Epoch 121][50/66] loss=0.913925 avg=0.869715 VRAM=38.7GiB | 40.6% done | ETA(epoch): 307s + [Epoch 121][60/66] loss=0.862345 avg=0.864567 VRAM=38.6GiB | 40.6% done | ETA(epoch): 115s + Train loss: 0.862729 (1266.1s) ETA: 4175min + Val loss: 0.845551 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9809 t_0.4-0.6=0.8506 t_0.6-0.8=0.7185 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 122/299 (41% done) --- + [Epoch 122][10/66] loss=0.770397 avg=0.844221 VRAM=38.7GiB | 40.7% done | ETA(epoch): 1074s + [Epoch 122][20/66] loss=0.963255 avg=0.859415 VRAM=38.6GiB | 40.8% done | ETA(epoch): 883s + [Epoch 122][30/66] loss=0.819885 avg=0.849183 VRAM=38.7GiB | 40.8% done | ETA(epoch): 694s + [Epoch 122][40/66] loss=0.890258 avg=0.847065 VRAM=38.6GiB | 40.9% done | ETA(epoch): 500s + [Epoch 122][50/66] loss=0.719200 avg=0.849522 VRAM=38.7GiB | 40.9% done | ETA(epoch): 308s + [Epoch 122][60/66] loss=0.906860 avg=0.855196 VRAM=38.6GiB | 41.0% done | ETA(epoch): 115s + Train loss: 0.853975 (1269.1s) ETA: 4151min + Val loss: 0.855912 [t_0.0-0.2=1.0605 t_0.2-0.4=0.9940 t_0.4-0.6=0.8428 t_0.6-0.8=0.7203 t_0.8-1.0=0.6593] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 123/299 (41% done) --- + [Epoch 123][10/66] loss=0.804983 avg=0.850791 VRAM=38.7GiB | 41.1% done | ETA(epoch): 1072s + [Epoch 123][20/66] loss=0.894884 avg=0.866705 VRAM=38.6GiB | 41.1% done | ETA(epoch): 886s + [Epoch 123][30/66] loss=0.758751 avg=0.860466 VRAM=38.7GiB | 41.2% done | ETA(epoch): 692s + [Epoch 123][40/66] loss=1.029475 avg=0.863322 VRAM=38.6GiB | 41.2% done | ETA(epoch): 500s + [Epoch 123][50/66] loss=0.937271 avg=0.860312 VRAM=38.7GiB | 41.3% done | ETA(epoch): 307s + [Epoch 123][60/66] loss=0.903350 avg=0.858498 VRAM=38.6GiB | 41.3% done | ETA(epoch): 115s + Train loss: 0.859069 (1267.9s) ETA: 4126min + Val loss: 0.855692 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9554 t_0.4-0.6=0.8566 t_0.6-0.8=0.7194 t_0.8-1.0=0.6494] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 124/299 (41% done) --- + [Epoch 124][10/66] loss=0.795019 avg=0.875205 VRAM=38.7GiB | 41.4% done | ETA(epoch): 1074s + [Epoch 124][20/66] loss=0.771340 avg=0.830329 VRAM=38.6GiB | 41.4% done | ETA(epoch): 882s + [Epoch 124][30/66] loss=0.684459 avg=0.830977 VRAM=38.7GiB | 41.5% done | ETA(epoch): 690s + [Epoch 124][40/66] loss=0.755129 avg=0.839055 VRAM=38.6GiB | 41.5% done | ETA(epoch): 499s + [Epoch 124][50/66] loss=0.851861 avg=0.841508 VRAM=38.7GiB | 41.6% done | ETA(epoch): 307s + [Epoch 124][60/66] loss=0.876930 avg=0.845649 VRAM=38.6GiB | 41.6% done | ETA(epoch): 115s + Train loss: 0.844983 (1265.8s) ETA: 4102min + Val loss: 0.865209 [t_0.0-0.2=1.0855 t_0.2-0.4=0.9790 t_0.4-0.6=0.8515 t_0.6-0.8=0.7197 t_0.8-1.0=0.6635] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 125/299 (42% done) --- + [Epoch 125][10/66] loss=0.987453 avg=0.866674 VRAM=38.7GiB | 41.7% done | ETA(epoch): 1087s + [Epoch 125][20/66] loss=0.914185 avg=0.852200 VRAM=38.6GiB | 41.8% done | ETA(epoch): 888s + [Epoch 125][30/66] loss=0.928623 avg=0.870312 VRAM=38.7GiB | 41.8% done | ETA(epoch): 694s + [Epoch 125][40/66] loss=0.827870 avg=0.864755 VRAM=38.6GiB | 41.9% done | ETA(epoch): 501s + [Epoch 125][50/66] loss=0.901121 avg=0.858949 VRAM=38.7GiB | 41.9% done | ETA(epoch): 308s + [Epoch 125][60/66] loss=0.754500 avg=0.858430 VRAM=38.6GiB | 42.0% done | ETA(epoch): 115s + Train loss: 0.853595 (1271.3s) ETA: 4077min + Val loss: 0.830618 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9507 t_0.4-0.6=0.8335 t_0.6-0.8=0.7323 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 126/299 (42% done) --- + [Epoch 126][10/66] loss=0.985721 avg=0.838734 VRAM=38.7GiB | 42.1% done | ETA(epoch): 1075s + [Epoch 126][20/66] loss=0.990719 avg=0.842516 VRAM=38.6GiB | 42.1% done | ETA(epoch): 883s + [Epoch 126][30/66] loss=0.813207 avg=0.835771 VRAM=38.7GiB | 42.2% done | ETA(epoch): 691s + [Epoch 126][40/66] loss=0.774008 avg=0.848560 VRAM=38.6GiB | 42.2% done | ETA(epoch): 499s + [Epoch 126][50/66] loss=0.854142 avg=0.854970 VRAM=38.7GiB | 42.3% done | ETA(epoch): 307s + [Epoch 126][60/66] loss=0.757393 avg=0.851503 VRAM=38.6GiB | 42.3% done | ETA(epoch): 115s + Train loss: 0.854778 (1268.1s) ETA: 4053min + Val loss: 0.813519 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9818 t_0.4-0.6=0.8292 t_0.6-0.8=0.7071 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 127/299 (42% done) --- + [Epoch 127][10/66] loss=0.963557 avg=0.877266 VRAM=38.7GiB | 42.4% done | ETA(epoch): 1074s + [Epoch 127][20/66] loss=0.887115 avg=0.859309 VRAM=38.6GiB | 42.4% done | ETA(epoch): 882s + [Epoch 127][30/66] loss=0.947664 avg=0.866036 VRAM=38.7GiB | 42.5% done | ETA(epoch): 691s + [Epoch 127][40/66] loss=0.785834 avg=0.857445 VRAM=38.6GiB | 42.5% done | ETA(epoch): 499s + [Epoch 127][50/66] loss=0.828912 avg=0.856188 VRAM=38.7GiB | 42.6% done | ETA(epoch): 309s + [Epoch 127][60/66] loss=0.839030 avg=0.852208 VRAM=38.6GiB | 42.6% done | ETA(epoch): 116s + Train loss: 0.854927 (1271.6s) ETA: 4028min + Val loss: 0.857016 [t_0.0-0.2=1.0761 t_0.2-0.4=1.0175 t_0.4-0.6=0.8519 t_0.6-0.8=0.7159 t_0.8-1.0=0.6498] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 128/299 (43% done) --- + [Epoch 128][10/66] loss=0.807672 avg=0.836722 VRAM=38.7GiB | 42.7% done | ETA(epoch): 1076s + [Epoch 128][20/66] loss=0.836284 avg=0.855325 VRAM=38.6GiB | 42.8% done | ETA(epoch): 883s + [Epoch 128][30/66] loss=0.826950 avg=0.848386 VRAM=38.7GiB | 42.8% done | ETA(epoch): 691s + [Epoch 128][40/66] loss=0.920138 avg=0.851954 VRAM=38.6GiB | 42.9% done | ETA(epoch): 499s + [Epoch 128][50/66] loss=0.756851 avg=0.854241 VRAM=38.7GiB | 42.9% done | ETA(epoch): 307s + [Epoch 128][60/66] loss=0.864309 avg=0.853280 VRAM=38.6GiB | 43.0% done | ETA(epoch): 115s + Train loss: 0.850907 (1267.0s) ETA: 4004min + Val loss: 0.848392 [t_0.0-0.2=1.0518 t_0.2-0.4=1.0008 t_0.4-0.6=0.8365 t_0.6-0.8=0.7270 t_0.8-1.0=0.6523] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 129/299 (43% done) --- + [Epoch 129][10/66] loss=0.894170 avg=0.830572 VRAM=38.7GiB | 43.1% done | ETA(epoch): 1075s + [Epoch 129][20/66] loss=0.768513 avg=0.834303 VRAM=38.6GiB | 43.1% done | ETA(epoch): 883s + [Epoch 129][30/66] loss=0.837907 avg=0.836937 VRAM=38.7GiB | 43.2% done | ETA(epoch): 692s + [Epoch 129][40/66] loss=0.865709 avg=0.841577 VRAM=38.6GiB | 43.2% done | ETA(epoch): 501s + [Epoch 129][50/66] loss=0.863172 avg=0.848108 VRAM=38.7GiB | 43.3% done | ETA(epoch): 308s + [Epoch 129][60/66] loss=0.809796 avg=0.841594 VRAM=38.6GiB | 43.3% done | ETA(epoch): 116s + Train loss: 0.841696 (1270.3s) ETA: 3980min + Val loss: 0.906046 [t_0.0-0.2=1.0720 t_0.2-0.4=0.9977 t_0.4-0.6=0.8841 t_0.6-0.8=0.6905 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 130/299 (43% done) --- + [Epoch 130][10/66] loss=0.863670 avg=0.875507 VRAM=38.7GiB | 43.4% done | ETA(epoch): 1076s + [Epoch 130][20/66] loss=0.798763 avg=0.884654 VRAM=38.6GiB | 43.4% done | ETA(epoch): 883s + [Epoch 130][30/66] loss=0.726157 avg=0.878169 VRAM=38.7GiB | 43.5% done | ETA(epoch): 697s + [Epoch 130][40/66] loss=0.876514 avg=0.876855 VRAM=38.6GiB | 43.5% done | ETA(epoch): 503s + [Epoch 130][50/66] loss=0.844097 avg=0.872824 VRAM=38.7GiB | 43.6% done | ETA(epoch): 309s + [Epoch 130][60/66] loss=0.956206 avg=0.868426 VRAM=38.6GiB | 43.6% done | ETA(epoch): 116s + Train loss: 0.863092 (1272.8s) ETA: 3956min + Val loss: 0.876184 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9870 t_0.4-0.6=0.8622 t_0.6-0.8=0.7285 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 131/299 (44% done) --- + [Epoch 131][10/66] loss=0.989698 avg=0.852127 VRAM=38.7GiB | 43.7% done | ETA(epoch): 1074s + [Epoch 131][20/66] loss=0.925231 avg=0.857236 VRAM=38.6GiB | 43.8% done | ETA(epoch): 890s + [Epoch 131][30/66] loss=1.016544 avg=0.859000 VRAM=38.7GiB | 43.8% done | ETA(epoch): 694s + [Epoch 131][40/66] loss=0.842433 avg=0.858440 VRAM=38.6GiB | 43.9% done | ETA(epoch): 501s + [Epoch 131][50/66] loss=0.902202 avg=0.858833 VRAM=38.7GiB | 43.9% done | ETA(epoch): 308s + [Epoch 131][60/66] loss=0.941536 avg=0.862611 VRAM=38.6GiB | 44.0% done | ETA(epoch): 116s + Train loss: 0.860191 (1270.6s) ETA: 3931min + Val loss: 0.865831 [t_0.0-0.2=1.0804 t_0.2-0.4=1.0167 t_0.4-0.6=0.7910 t_0.6-0.8=0.7032 t_0.8-1.0=0.6652] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 132/299 (44% done) --- + [Epoch 132][10/66] loss=0.800356 avg=0.845113 VRAM=38.7GiB | 44.1% done | ETA(epoch): 1087s + [Epoch 132][20/66] loss=0.859781 avg=0.848942 VRAM=38.6GiB | 44.1% done | ETA(epoch): 888s + [Epoch 132][30/66] loss=0.991624 avg=0.847441 VRAM=38.7GiB | 44.2% done | ETA(epoch): 694s + [Epoch 132][40/66] loss=0.972825 avg=0.848870 VRAM=38.6GiB | 44.2% done | ETA(epoch): 501s + [Epoch 132][50/66] loss=0.883914 avg=0.848069 VRAM=38.7GiB | 44.3% done | ETA(epoch): 308s + [Epoch 132][60/66] loss=0.891214 avg=0.843566 VRAM=38.6GiB | 44.3% done | ETA(epoch): 115s + Train loss: 0.846584 (1269.0s) ETA: 3907min + Val loss: 0.889135 [t_0.0-0.2=1.0570 t_0.2-0.4=0.9999 t_0.4-0.6=0.8855 t_0.6-0.8=0.7102 t_0.8-1.0=0.6575] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 133/299 (44% done) --- + [Epoch 133][10/66] loss=0.956566 avg=0.860314 VRAM=38.7GiB | 44.4% done | ETA(epoch): 1073s + [Epoch 133][20/66] loss=0.850479 avg=0.867388 VRAM=38.6GiB | 44.4% done | ETA(epoch): 884s + [Epoch 133][30/66] loss=0.899327 avg=0.868252 VRAM=38.7GiB | 44.5% done | ETA(epoch): 691s + [Epoch 133][40/66] loss=0.891707 avg=0.860885 VRAM=38.6GiB | 44.5% done | ETA(epoch): 499s + [Epoch 133][50/66] loss=0.836573 avg=0.861286 VRAM=38.7GiB | 44.6% done | ETA(epoch): 307s + [Epoch 133][60/66] loss=0.888073 avg=0.866372 VRAM=38.6GiB | 44.6% done | ETA(epoch): 115s + Train loss: 0.865684 (1268.8s) ETA: 3883min + Val loss: 0.862414 [t_0.0-0.2=1.0861 t_0.2-0.4=0.9537 t_0.4-0.6=0.8348 t_0.6-0.8=0.7298 t_0.8-1.0=0.6493] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 134/299 (45% done) --- + [Epoch 134][10/66] loss=0.991771 avg=0.887274 VRAM=38.7GiB | 44.7% done | ETA(epoch): 1073s + [Epoch 134][20/66] loss=0.738085 avg=0.845196 VRAM=38.6GiB | 44.8% done | ETA(epoch): 884s + [Epoch 134][30/66] loss=0.840095 avg=0.843806 VRAM=38.7GiB | 44.8% done | ETA(epoch): 692s + [Epoch 134][40/66] loss=0.859215 avg=0.852589 VRAM=38.6GiB | 44.9% done | ETA(epoch): 500s + [Epoch 134][50/66] loss=1.001384 avg=0.861500 VRAM=38.7GiB | 44.9% done | ETA(epoch): 307s + [Epoch 134][60/66] loss=0.743269 avg=0.864685 VRAM=38.6GiB | 45.0% done | ETA(epoch): 115s + Train loss: 0.866645 (1267.8s) ETA: 3859min + Val loss: 0.847219 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9758 t_0.4-0.6=0.8550 t_0.6-0.8=0.7193 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 135/299 (45% done) --- + [Epoch 135][10/66] loss=0.853054 avg=0.902341 VRAM=38.7GiB | 45.1% done | ETA(epoch): 1075s + [Epoch 135][20/66] loss=0.905486 avg=0.883891 VRAM=38.6GiB | 45.1% done | ETA(epoch): 883s + [Epoch 135][30/66] loss=0.795568 avg=0.871350 VRAM=38.7GiB | 45.2% done | ETA(epoch): 692s + [Epoch 135][40/66] loss=0.869360 avg=0.864799 VRAM=38.6GiB | 45.2% done | ETA(epoch): 499s + [Epoch 135][50/66] loss=0.736750 avg=0.865950 VRAM=38.7GiB | 45.3% done | ETA(epoch): 308s + [Epoch 135][60/66] loss=0.878589 avg=0.861073 VRAM=38.6GiB | 45.3% done | ETA(epoch): 115s + Train loss: 0.864080 (1270.0s) ETA: 3835min + Val loss: 0.838219 [t_0.0-0.2=1.0664 t_0.2-0.4=0.9653 t_0.4-0.6=0.8350 t_0.6-0.8=0.7334 t_0.8-1.0=0.6441] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 136/299 (45% done) --- + [Epoch 136][10/66] loss=0.959617 avg=0.863186 VRAM=38.7GiB | 45.4% done | ETA(epoch): 1073s + [Epoch 136][20/66] loss=0.826728 avg=0.840784 VRAM=38.6GiB | 45.4% done | ETA(epoch): 883s + [Epoch 136][30/66] loss=0.848014 avg=0.849039 VRAM=38.7GiB | 45.5% done | ETA(epoch): 691s + [Epoch 136][40/66] loss=0.881444 avg=0.861057 VRAM=38.6GiB | 45.5% done | ETA(epoch): 500s + [Epoch 136][50/66] loss=0.788363 avg=0.856458 VRAM=38.7GiB | 45.6% done | ETA(epoch): 308s + [Epoch 136][60/66] loss=0.982032 avg=0.852112 VRAM=38.6GiB | 45.6% done | ETA(epoch): 115s + Train loss: 0.850879 (1269.3s) ETA: 3811min + Val loss: 0.861876 [t_0.0-0.2=1.0691 t_0.2-0.4=0.9963 t_0.4-0.6=0.8482 t_0.6-0.8=0.7130 t_0.8-1.0=0.6488] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 137/299 (46% done) --- + [Epoch 137][10/66] loss=0.818063 avg=0.861223 VRAM=38.7GiB | 45.7% done | ETA(epoch): 1074s + [Epoch 137][20/66] loss=0.907209 avg=0.857894 VRAM=38.6GiB | 45.8% done | ETA(epoch): 883s + [Epoch 137][30/66] loss=0.862026 avg=0.865609 VRAM=38.7GiB | 45.8% done | ETA(epoch): 693s + [Epoch 137][40/66] loss=0.805498 avg=0.861224 VRAM=38.6GiB | 45.9% done | ETA(epoch): 500s + [Epoch 137][50/66] loss=0.887009 avg=0.864779 VRAM=38.7GiB | 45.9% done | ETA(epoch): 308s + [Epoch 137][60/66] loss=0.870269 avg=0.865492 VRAM=38.6GiB | 46.0% done | ETA(epoch): 115s + Train loss: 0.867499 (1268.8s) ETA: 3787min + Val loss: 0.831874 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9800 t_0.4-0.6=0.8559 t_0.6-0.8=0.7098 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 138/299 (46% done) --- + [Epoch 138][10/66] loss=0.902750 avg=0.904934 VRAM=38.7GiB | 46.1% done | ETA(epoch): 1075s + [Epoch 138][20/66] loss=0.905828 avg=0.869972 VRAM=38.6GiB | 46.1% done | ETA(epoch): 882s + [Epoch 138][30/66] loss=0.899193 avg=0.870966 VRAM=38.7GiB | 46.2% done | ETA(epoch): 693s + [Epoch 138][40/66] loss=0.839735 avg=0.873389 VRAM=38.6GiB | 46.2% done | ETA(epoch): 500s + [Epoch 138][50/66] loss=0.848969 avg=0.870884 VRAM=38.7GiB | 46.3% done | ETA(epoch): 308s + [Epoch 138][60/66] loss=0.864850 avg=0.866950 VRAM=38.6GiB | 46.3% done | ETA(epoch): 115s + Train loss: 0.870549 (1269.1s) ETA: 3762min + Val loss: 0.839346 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9866 t_0.4-0.6=0.8293 t_0.6-0.8=0.7095 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 139/299 (46% done) --- + [Epoch 139][10/66] loss=0.901214 avg=0.862231 VRAM=38.7GiB | 46.4% done | ETA(epoch): 1075s + [Epoch 139][20/66] loss=0.851425 avg=0.845783 VRAM=38.6GiB | 46.4% done | ETA(epoch): 891s + [Epoch 139][30/66] loss=0.812633 avg=0.851471 VRAM=38.7GiB | 46.5% done | ETA(epoch): 695s + [Epoch 139][40/66] loss=0.849438 avg=0.855949 VRAM=38.6GiB | 46.5% done | ETA(epoch): 502s + [Epoch 139][50/66] loss=0.855815 avg=0.852220 VRAM=38.7GiB | 46.6% done | ETA(epoch): 308s + [Epoch 139][60/66] loss=0.968666 avg=0.851877 VRAM=38.6GiB | 46.6% done | ETA(epoch): 116s + Train loss: 0.850227 (1270.5s) ETA: 3738min + Val loss: 0.873336 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9991 t_0.4-0.6=0.8500 t_0.6-0.8=0.7248 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 140/299 (47% done) --- + [Epoch 140][10/66] loss=0.901799 avg=0.871104 VRAM=38.7GiB | 46.7% done | ETA(epoch): 1085s + [Epoch 140][20/66] loss=0.948411 avg=0.864758 VRAM=38.6GiB | 46.8% done | ETA(epoch): 888s + [Epoch 140][30/66] loss=0.940087 avg=0.862760 VRAM=38.7GiB | 46.8% done | ETA(epoch): 694s + [Epoch 140][40/66] loss=0.891742 avg=0.871186 VRAM=38.6GiB | 46.9% done | ETA(epoch): 501s + [Epoch 140][50/66] loss=0.888804 avg=0.877870 VRAM=38.7GiB | 46.9% done | ETA(epoch): 308s + [Epoch 140][60/66] loss=0.837074 avg=0.872491 VRAM=38.6GiB | 47.0% done | ETA(epoch): 115s + Train loss: 0.871418 (1269.3s) ETA: 3714min + Val loss: 0.864244 [t_0.0-0.2=1.0589 t_0.2-0.4=1.0029 t_0.4-0.6=0.8275 t_0.6-0.8=0.7140 t_0.8-1.0=0.6402] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 141/299 (47% done) --- + [Epoch 141][10/66] loss=0.912558 avg=0.860790 VRAM=38.7GiB | 47.1% done | ETA(epoch): 1073s + [Epoch 141][20/66] loss=0.809460 avg=0.857536 VRAM=38.6GiB | 47.1% done | ETA(epoch): 883s + [Epoch 141][30/66] loss=0.834502 avg=0.863831 VRAM=38.7GiB | 47.2% done | ETA(epoch): 691s + [Epoch 141][40/66] loss=0.795306 avg=0.857551 VRAM=38.6GiB | 47.2% done | ETA(epoch): 499s + [Epoch 141][50/66] loss=0.796482 avg=0.858078 VRAM=38.7GiB | 47.3% done | ETA(epoch): 307s + [Epoch 141][60/66] loss=0.764161 avg=0.854799 VRAM=38.6GiB | 47.3% done | ETA(epoch): 117s + Train loss: 0.857574 (1285.3s) ETA: 3691min + Val loss: 0.855473 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9940 t_0.4-0.6=0.8742 t_0.6-0.8=0.7143 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 142/299 (47% done) --- + [Epoch 142][10/66] loss=0.875233 avg=0.892676 VRAM=38.7GiB | 47.4% done | ETA(epoch): 1074s + [Epoch 142][20/66] loss=0.961545 avg=0.882306 VRAM=38.6GiB | 47.4% done | ETA(epoch): 883s + [Epoch 142][30/66] loss=0.860704 avg=0.863264 VRAM=38.7GiB | 47.5% done | ETA(epoch): 691s + [Epoch 142][40/66] loss=0.830947 avg=0.864304 VRAM=38.6GiB | 47.5% done | ETA(epoch): 499s + [Epoch 142][50/66] loss=0.727884 avg=0.854029 VRAM=38.7GiB | 47.6% done | ETA(epoch): 307s + [Epoch 142][60/66] loss=0.895241 avg=0.855101 VRAM=38.6GiB | 47.6% done | ETA(epoch): 117s + Train loss: 0.856362 (1285.5s) ETA: 3667min + Val loss: 0.864323 [t_0.0-0.2=1.0784 t_0.2-0.4=0.9816 t_0.4-0.6=0.8297 t_0.6-0.8=0.7064 t_0.8-1.0=0.6383] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 143/299 (48% done) --- + [Epoch 143][10/66] loss=0.808544 avg=0.868695 VRAM=38.7GiB | 47.7% done | ETA(epoch): 1075s + [Epoch 143][20/66] loss=0.930405 avg=0.856382 VRAM=38.6GiB | 47.8% done | ETA(epoch): 883s + [Epoch 143][30/66] loss=0.805414 avg=0.849221 VRAM=38.7GiB | 47.8% done | ETA(epoch): 691s + [Epoch 143][40/66] loss=0.811764 avg=0.853111 VRAM=38.6GiB | 47.9% done | ETA(epoch): 499s + [Epoch 143][50/66] loss=0.904735 avg=0.860581 VRAM=38.7GiB | 47.9% done | ETA(epoch): 308s + [Epoch 143][60/66] loss=0.770185 avg=0.858048 VRAM=38.6GiB | 48.0% done | ETA(epoch): 116s + Train loss: 0.860843 (1270.4s) ETA: 3643min + Val loss: 0.865234 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9750 t_0.4-0.6=0.8366 t_0.6-0.8=0.7300 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 144/299 (48% done) --- + [Epoch 144][10/66] loss=0.821149 avg=0.855223 VRAM=38.7GiB | 48.1% done | ETA(epoch): 1074s + [Epoch 144][20/66] loss=0.845649 avg=0.867670 VRAM=38.6GiB | 48.1% done | ETA(epoch): 884s + [Epoch 144][30/66] loss=0.912701 avg=0.864565 VRAM=38.7GiB | 48.2% done | ETA(epoch): 692s + [Epoch 144][40/66] loss=0.927282 avg=0.857386 VRAM=38.6GiB | 48.2% done | ETA(epoch): 499s + [Epoch 144][50/66] loss=0.930997 avg=0.862636 VRAM=38.7GiB | 48.3% done | ETA(epoch): 307s + [Epoch 144][60/66] loss=0.857192 avg=0.858939 VRAM=38.6GiB | 48.3% done | ETA(epoch): 115s + Train loss: 0.858884 (1267.8s) ETA: 3619min + Val loss: 0.887668 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0042 t_0.4-0.6=0.8431 t_0.6-0.8=0.6993 t_0.8-1.0=0.6358] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 145/299 (48% done) --- + [Epoch 145][10/66] loss=0.737408 avg=0.850869 VRAM=38.7GiB | 48.4% done | ETA(epoch): 1073s + [Epoch 145][20/66] loss=0.937545 avg=0.864640 VRAM=38.6GiB | 48.4% done | ETA(epoch): 882s + [Epoch 145][30/66] loss=0.814080 avg=0.866551 VRAM=38.7GiB | 48.5% done | ETA(epoch): 691s + [Epoch 145][40/66] loss=0.810177 avg=0.864017 VRAM=38.6GiB | 48.5% done | ETA(epoch): 499s + [Epoch 145][50/66] loss=0.971861 avg=0.860656 VRAM=38.7GiB | 48.6% done | ETA(epoch): 307s + [Epoch 145][60/66] loss=0.952706 avg=0.862345 VRAM=38.6GiB | 48.6% done | ETA(epoch): 115s + Train loss: 0.861790 (1267.1s) ETA: 3595min + Val loss: 0.879557 [t_0.0-0.2=1.0913 t_0.2-0.4=0.9672 t_0.4-0.6=0.8505 t_0.6-0.8=0.7272 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 146/299 (49% done) --- + [Epoch 146][10/66] loss=0.847369 avg=0.868453 VRAM=38.7GiB | 48.7% done | ETA(epoch): 1075s + [Epoch 146][20/66] loss=0.893762 avg=0.863773 VRAM=38.6GiB | 48.8% done | ETA(epoch): 883s + [Epoch 146][30/66] loss=0.938526 avg=0.861471 VRAM=38.7GiB | 48.8% done | ETA(epoch): 691s + [Epoch 146][40/66] loss=0.902970 avg=0.858971 VRAM=38.6GiB | 48.9% done | ETA(epoch): 499s + [Epoch 146][50/66] loss=0.826448 avg=0.858673 VRAM=38.7GiB | 48.9% done | ETA(epoch): 307s + [Epoch 146][60/66] loss=0.758255 avg=0.860311 VRAM=38.6GiB | 49.0% done | ETA(epoch): 115s + Train loss: 0.861665 (1267.5s) ETA: 3571min + Val loss: 0.837710 [t_0.0-0.2=1.0527 t_0.2-0.4=0.9970 t_0.4-0.6=0.8778 t_0.6-0.8=0.7042 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 147/299 (49% done) --- + [Epoch 147][10/66] loss=0.746488 avg=0.858908 VRAM=38.7GiB | 49.1% done | ETA(epoch): 1089s + [Epoch 147][20/66] loss=0.891003 avg=0.860095 VRAM=38.6GiB | 49.1% done | ETA(epoch): 889s + [Epoch 147][30/66] loss=0.739864 avg=0.852898 VRAM=38.7GiB | 49.2% done | ETA(epoch): 694s + [Epoch 147][40/66] loss=0.796761 avg=0.851988 VRAM=38.6GiB | 49.2% done | ETA(epoch): 501s + [Epoch 147][50/66] loss=0.975580 avg=0.858432 VRAM=38.7GiB | 49.3% done | ETA(epoch): 308s + [Epoch 147][60/66] loss=0.843401 avg=0.862852 VRAM=38.6GiB | 49.3% done | ETA(epoch): 115s + Train loss: 0.863965 (1270.0s) ETA: 3547min + Val loss: 0.874916 [t_0.0-0.2=1.0719 t_0.2-0.4=1.0474 t_0.4-0.6=0.8285 t_0.6-0.8=0.6820 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 148/299 (49% done) --- + [Epoch 148][10/66] loss=0.884268 avg=0.853974 VRAM=38.7GiB | 49.4% done | ETA(epoch): 1074s + [Epoch 148][20/66] loss=0.843240 avg=0.866986 VRAM=38.6GiB | 49.4% done | ETA(epoch): 883s + [Epoch 148][30/66] loss=0.788094 avg=0.861633 VRAM=38.7GiB | 49.5% done | ETA(epoch): 691s + [Epoch 148][40/66] loss=0.859163 avg=0.866014 VRAM=38.6GiB | 49.5% done | ETA(epoch): 499s + [Epoch 148][50/66] loss=0.891541 avg=0.864553 VRAM=38.7GiB | 49.6% done | ETA(epoch): 307s + [Epoch 148][60/66] loss=0.855271 avg=0.866418 VRAM=38.6GiB | 49.6% done | ETA(epoch): 115s + Train loss: 0.864066 (1267.1s) ETA: 3523min + Val loss: 0.848182 [t_0.0-0.2=1.0600 t_0.2-0.4=0.9788 t_0.4-0.6=0.8348 t_0.6-0.8=0.7183 t_0.8-1.0=0.6292] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 149/299 (50% done) --- + [Epoch 149][10/66] loss=0.844031 avg=0.853861 VRAM=38.7GiB | 49.7% done | ETA(epoch): 1075s + [Epoch 149][20/66] loss=0.800901 avg=0.858420 VRAM=38.6GiB | 49.8% done | ETA(epoch): 883s + [Epoch 149][30/66] loss=0.722244 avg=0.867250 VRAM=38.7GiB | 49.8% done | ETA(epoch): 691s + [Epoch 149][40/66] loss=0.881815 avg=0.864655 VRAM=38.6GiB | 49.9% done | ETA(epoch): 499s + [Epoch 149][50/66] loss=0.861863 avg=0.859341 VRAM=38.7GiB | 49.9% done | ETA(epoch): 307s + [Epoch 149][60/66] loss=0.995830 avg=0.861017 VRAM=38.6GiB | 50.0% done | ETA(epoch): 115s + Train loss: 0.858984 (1267.5s) ETA: 3499min + Val loss: 0.894839 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9771 t_0.4-0.6=0.8641 t_0.6-0.8=0.7308 t_0.8-1.0=0.6453] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 150/299 (50% done) --- + [MilestoneVis] train_0 step 9900 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9900 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9900 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9900 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9900 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9900 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9900 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9900 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9900 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9900 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/66] loss=0.838068 avg=0.881550 VRAM=38.7GiB | 50.1% done | ETA(epoch): 6871s + [Epoch 150][20/66] loss=0.816628 avg=0.855871 VRAM=38.6GiB | 50.1% done | ETA(epoch): 3263s + [Epoch 150][30/66] loss=0.958540 avg=0.858314 VRAM=38.7GiB | 50.2% done | ETA(epoch): 1932s + [Epoch 150][40/66] loss=0.797938 avg=0.864659 VRAM=38.6GiB | 50.2% done | ETA(epoch): 1171s + [Epoch 150][50/66] loss=0.833752 avg=0.860702 VRAM=38.7GiB | 50.3% done | ETA(epoch): 638s + [Epoch 150][60/66] loss=0.940688 avg=0.857925 VRAM=38.6GiB | 50.3% done | ETA(epoch): 219s + Train loss: 0.857955 (2303.5s) ETA: 3492min + Val loss: 0.846838 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0094 t_0.4-0.6=0.8416 t_0.6-0.8=0.7149 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 151/299 (50% done) --- + [Epoch 151][10/66] loss=0.804360 avg=0.829310 VRAM=38.7GiB | 50.4% done | ETA(epoch): 1074s + [Epoch 151][20/66] loss=0.947116 avg=0.843487 VRAM=38.6GiB | 50.4% done | ETA(epoch): 883s + [Epoch 151][30/66] loss=0.769513 avg=0.834911 VRAM=38.7GiB | 50.5% done | ETA(epoch): 691s + [Epoch 151][40/66] loss=0.840649 avg=0.840960 VRAM=38.6GiB | 50.5% done | ETA(epoch): 499s + [Epoch 151][50/66] loss=0.901710 avg=0.841340 VRAM=38.7GiB | 50.6% done | ETA(epoch): 308s + [Epoch 151][60/66] loss=0.844185 avg=0.843306 VRAM=38.6GiB | 50.6% done | ETA(epoch): 115s + Train loss: 0.843377 (1268.7s) ETA: 3468min + Val loss: 0.856354 [t_0.0-0.2=1.0746 t_0.2-0.4=1.0054 t_0.4-0.6=0.8343 t_0.6-0.8=0.6963 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 152/299 (51% done) --- + [Epoch 152][10/66] loss=0.846585 avg=0.854807 VRAM=38.7GiB | 50.7% done | ETA(epoch): 1074s + [Epoch 152][20/66] loss=0.896288 avg=0.857008 VRAM=38.6GiB | 50.8% done | ETA(epoch): 883s + [Epoch 152][30/66] loss=0.816864 avg=0.855752 VRAM=38.7GiB | 50.8% done | ETA(epoch): 691s + [Epoch 152][40/66] loss=0.988895 avg=0.855990 VRAM=38.6GiB | 50.9% done | ETA(epoch): 499s + [Epoch 152][50/66] loss=0.776015 avg=0.851934 VRAM=38.7GiB | 50.9% done | ETA(epoch): 307s + [Epoch 152][60/66] loss=0.913755 avg=0.851014 VRAM=38.6GiB | 51.0% done | ETA(epoch): 115s + Train loss: 0.856853 (1266.6s) ETA: 3444min + Val loss: 0.835889 [t_0.0-0.2=1.0929 t_0.2-0.4=1.0025 t_0.4-0.6=0.8207 t_0.6-0.8=0.7152 t_0.8-1.0=0.6466] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 153/299 (51% done) --- + [Epoch 153][10/66] loss=0.926080 avg=0.888444 VRAM=38.7GiB | 51.1% done | ETA(epoch): 1078s + [Epoch 153][20/66] loss=0.893441 avg=0.896440 VRAM=38.6GiB | 51.1% done | ETA(epoch): 884s + [Epoch 153][30/66] loss=0.915100 avg=0.880670 VRAM=38.7GiB | 51.2% done | ETA(epoch): 692s + [Epoch 153][40/66] loss=0.792522 avg=0.876924 VRAM=38.6GiB | 51.2% done | ETA(epoch): 501s + [Epoch 153][50/66] loss=0.805794 avg=0.867230 VRAM=38.7GiB | 51.3% done | ETA(epoch): 308s + [Epoch 153][60/66] loss=0.911154 avg=0.870351 VRAM=38.6GiB | 51.3% done | ETA(epoch): 115s + Train loss: 0.870370 (1269.8s) ETA: 3420min + Val loss: 0.834319 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9750 t_0.4-0.6=0.8271 t_0.6-0.8=0.7100 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 154/299 (51% done) --- + [Epoch 154][10/66] loss=0.883493 avg=0.821877 VRAM=38.7GiB | 51.4% done | ETA(epoch): 1073s + [Epoch 154][20/66] loss=0.733045 avg=0.821042 VRAM=38.6GiB | 51.4% done | ETA(epoch): 882s + [Epoch 154][30/66] loss=0.883751 avg=0.824927 VRAM=38.7GiB | 51.5% done | ETA(epoch): 691s + [Epoch 154][40/66] loss=0.916192 avg=0.839959 VRAM=38.6GiB | 51.5% done | ETA(epoch): 499s + [Epoch 154][50/66] loss=0.940169 avg=0.851316 VRAM=38.7GiB | 51.6% done | ETA(epoch): 307s + [Epoch 154][60/66] loss=0.808320 avg=0.851731 VRAM=38.6GiB | 51.6% done | ETA(epoch): 115s + Train loss: 0.851584 (1267.4s) ETA: 3396min + Val loss: 0.859099 [t_0.0-0.2=1.0813 t_0.2-0.4=1.0336 t_0.4-0.6=0.8298 t_0.6-0.8=0.7247 t_0.8-1.0=0.6570] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 155/299 (52% done) --- + [Epoch 155][10/66] loss=0.762682 avg=0.862605 VRAM=38.7GiB | 51.7% done | ETA(epoch): 1076s + [Epoch 155][20/66] loss=0.910397 avg=0.860523 VRAM=38.6GiB | 51.8% done | ETA(epoch): 884s + [Epoch 155][30/66] loss=0.798733 avg=0.856137 VRAM=38.7GiB | 51.8% done | ETA(epoch): 691s + [Epoch 155][40/66] loss=0.768300 avg=0.852359 VRAM=38.6GiB | 51.9% done | ETA(epoch): 499s + [Epoch 155][50/66] loss=0.915288 avg=0.858707 VRAM=38.7GiB | 51.9% done | ETA(epoch): 307s + [Epoch 155][60/66] loss=0.871054 avg=0.860693 VRAM=38.6GiB | 52.0% done | ETA(epoch): 115s + Train loss: 0.855476 (1267.2s) ETA: 3371min + Val loss: 0.869013 [t_0.0-0.2=1.0786 t_0.2-0.4=1.0072 t_0.4-0.6=0.8656 t_0.6-0.8=0.7126 t_0.8-1.0=0.6438] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 156/299 (52% done) --- + [Epoch 156][10/66] loss=0.868720 avg=0.836014 VRAM=38.7GiB | 52.1% done | ETA(epoch): 1086s + [Epoch 156][20/66] loss=0.803103 avg=0.829413 VRAM=38.6GiB | 52.1% done | ETA(epoch): 887s + [Epoch 156][30/66] loss=0.968656 avg=0.836226 VRAM=38.7GiB | 52.2% done | ETA(epoch): 693s + [Epoch 156][40/66] loss=0.819499 avg=0.828660 VRAM=38.6GiB | 52.2% done | ETA(epoch): 500s + [Epoch 156][50/66] loss=0.759817 avg=0.839866 VRAM=38.7GiB | 52.3% done | ETA(epoch): 308s + [Epoch 156][60/66] loss=0.868894 avg=0.844221 VRAM=38.6GiB | 52.3% done | ETA(epoch): 115s + Train loss: 0.845542 (1268.3s) ETA: 3347min + Val loss: 0.854805 [t_0.0-0.2=1.0636 t_0.2-0.4=0.9765 t_0.4-0.6=0.8122 t_0.6-0.8=0.7595 t_0.8-1.0=0.6378] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 157/299 (52% done) --- + [Epoch 157][10/66] loss=0.707591 avg=0.846106 VRAM=38.7GiB | 52.4% done | ETA(epoch): 1076s + [Epoch 157][20/66] loss=0.780758 avg=0.855730 VRAM=38.6GiB | 52.4% done | ETA(epoch): 884s + [Epoch 157][30/66] loss=0.782860 avg=0.864969 VRAM=38.7GiB | 52.5% done | ETA(epoch): 691s + [Epoch 157][40/66] loss=0.837113 avg=0.860047 VRAM=38.6GiB | 52.5% done | ETA(epoch): 500s + [Epoch 157][50/66] loss=0.919963 avg=0.865121 VRAM=38.7GiB | 52.6% done | ETA(epoch): 307s + [Epoch 157][60/66] loss=0.751003 avg=0.861157 VRAM=38.6GiB | 52.6% done | ETA(epoch): 115s + Train loss: 0.857571 (1267.6s) ETA: 3323min + Val loss: 0.868392 [t_0.0-0.2=1.0922 t_0.2-0.4=0.9826 t_0.4-0.6=0.8610 t_0.6-0.8=0.7136 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 158/299 (53% done) --- + [Epoch 158][10/66] loss=0.741228 avg=0.855774 VRAM=38.7GiB | 52.7% done | ETA(epoch): 1076s + [Epoch 158][20/66] loss=0.869809 avg=0.849961 VRAM=38.6GiB | 52.8% done | ETA(epoch): 883s + [Epoch 158][30/66] loss=0.921165 avg=0.849036 VRAM=38.7GiB | 52.8% done | ETA(epoch): 691s + [Epoch 158][40/66] loss=0.915088 avg=0.856459 VRAM=38.6GiB | 52.9% done | ETA(epoch): 499s + [Epoch 158][50/66] loss=0.817827 avg=0.856183 VRAM=38.7GiB | 52.9% done | ETA(epoch): 307s + [Epoch 158][60/66] loss=0.779271 avg=0.860685 VRAM=38.6GiB | 53.0% done | ETA(epoch): 115s + Train loss: 0.856416 (1269.4s) ETA: 3299min + Val loss: 0.891796 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9802 t_0.4-0.6=0.8386 t_0.6-0.8=0.7206 t_0.8-1.0=0.6481] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 159/299 (53% done) --- + [Epoch 159][10/66] loss=0.898741 avg=0.835433 VRAM=38.7GiB | 53.1% done | ETA(epoch): 1075s + [Epoch 159][20/66] loss=0.848222 avg=0.866236 VRAM=38.6GiB | 53.1% done | ETA(epoch): 883s + [Epoch 159][30/66] loss=0.906501 avg=0.869421 VRAM=38.7GiB | 53.2% done | ETA(epoch): 691s + [Epoch 159][40/66] loss=0.852119 avg=0.862056 VRAM=38.6GiB | 53.2% done | ETA(epoch): 499s + [Epoch 159][50/66] loss=0.883728 avg=0.854560 VRAM=38.7GiB | 53.3% done | ETA(epoch): 307s + [Epoch 159][60/66] loss=0.938044 avg=0.851008 VRAM=38.6GiB | 53.3% done | ETA(epoch): 115s + Train loss: 0.853569 (1267.5s) ETA: 3275min + Val loss: 0.829764 [t_0.0-0.2=1.0777 t_0.2-0.4=0.9785 t_0.4-0.6=0.8423 t_0.6-0.8=0.7219 t_0.8-1.0=0.6542] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 160/299 (53% done) --- + [Epoch 160][10/66] loss=0.880304 avg=0.843756 VRAM=38.7GiB | 53.4% done | ETA(epoch): 1076s + [Epoch 160][20/66] loss=0.903366 avg=0.845336 VRAM=38.6GiB | 53.4% done | ETA(epoch): 883s + [Epoch 160][30/66] loss=0.738894 avg=0.856857 VRAM=38.7GiB | 53.5% done | ETA(epoch): 691s + [Epoch 160][40/66] loss=0.900806 avg=0.860332 VRAM=38.6GiB | 53.5% done | ETA(epoch): 500s + [Epoch 160][50/66] loss=0.750327 avg=0.858856 VRAM=38.7GiB | 53.6% done | ETA(epoch): 308s + [Epoch 160][60/66] loss=0.925989 avg=0.861054 VRAM=38.6GiB | 53.6% done | ETA(epoch): 115s + Train loss: 0.865141 (1268.9s) ETA: 3251min + Val loss: 0.854393 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9811 t_0.4-0.6=0.8676 t_0.6-0.8=0.7176 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 161/299 (54% done) --- + [Epoch 161][10/66] loss=0.741034 avg=0.848846 VRAM=38.7GiB | 53.7% done | ETA(epoch): 1075s + [Epoch 161][20/66] loss=0.838918 avg=0.853294 VRAM=38.6GiB | 53.8% done | ETA(epoch): 884s + [Epoch 161][30/66] loss=0.929287 avg=0.859388 VRAM=38.7GiB | 53.8% done | ETA(epoch): 692s + [Epoch 161][40/66] loss=0.909394 avg=0.867030 VRAM=38.6GiB | 53.9% done | ETA(epoch): 499s + [Epoch 161][50/66] loss=0.825425 avg=0.868346 VRAM=38.7GiB | 53.9% done | ETA(epoch): 307s + [Epoch 161][60/66] loss=0.909914 avg=0.862228 VRAM=38.6GiB | 54.0% done | ETA(epoch): 115s + Train loss: 0.858616 (1267.1s) ETA: 3227min + Val loss: 0.884881 [t_0.0-0.2=1.0644 t_0.2-0.4=0.9892 t_0.4-0.6=0.8456 t_0.6-0.8=0.7003 t_0.8-1.0=0.6648] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 162/299 (54% done) --- + [Epoch 162][10/66] loss=0.816809 avg=0.847957 VRAM=38.7GiB | 54.1% done | ETA(epoch): 1074s + [Epoch 162][20/66] loss=0.915873 avg=0.829990 VRAM=38.6GiB | 54.1% done | ETA(epoch): 890s + [Epoch 162][30/66] loss=0.912882 avg=0.840058 VRAM=38.7GiB | 54.2% done | ETA(epoch): 694s + [Epoch 162][40/66] loss=0.775688 avg=0.853261 VRAM=38.6GiB | 54.2% done | ETA(epoch): 501s + [Epoch 162][50/66] loss=0.892484 avg=0.857577 VRAM=38.7GiB | 54.3% done | ETA(epoch): 308s + [Epoch 162][60/66] loss=0.882275 avg=0.857024 VRAM=38.6GiB | 54.3% done | ETA(epoch): 115s + Train loss: 0.854503 (1270.2s) ETA: 3203min + Val loss: 0.869211 [t_0.0-0.2=1.0613 t_0.2-0.4=1.0214 t_0.4-0.6=0.8518 t_0.6-0.8=0.7155 t_0.8-1.0=0.6481] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 163/299 (54% done) --- + [Epoch 163][10/66] loss=0.883498 avg=0.852208 VRAM=38.7GiB | 54.4% done | ETA(epoch): 1075s + [Epoch 163][20/66] loss=0.836482 avg=0.862888 VRAM=38.6GiB | 54.4% done | ETA(epoch): 893s + [Epoch 163][30/66] loss=0.887033 avg=0.867682 VRAM=38.7GiB | 54.5% done | ETA(epoch): 696s + [Epoch 163][40/66] loss=0.828259 avg=0.870144 VRAM=38.6GiB | 54.5% done | ETA(epoch): 502s + [Epoch 163][50/66] loss=0.754036 avg=0.870305 VRAM=38.7GiB | 54.6% done | ETA(epoch): 309s + [Epoch 163][60/66] loss=0.751712 avg=0.870111 VRAM=38.6GiB | 54.6% done | ETA(epoch): 116s + Train loss: 0.866999 (1271.1s) ETA: 3180min + Val loss: 0.875448 [t_0.0-0.2=1.0596 t_0.2-0.4=1.0385 t_0.4-0.6=0.8578 t_0.6-0.8=0.7056 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 21.0/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 164/299 (55% done) --- + [Epoch 164][10/66] loss=0.900666 avg=0.841915 VRAM=38.7GiB | 54.7% done | ETA(epoch): 1074s + [Epoch 164][20/66] loss=0.766222 avg=0.847217 VRAM=38.6GiB | 54.8% done | ETA(epoch): 882s + [Epoch 164][30/66] loss=0.904670 avg=0.850974 VRAM=38.7GiB | 54.8% done | ETA(epoch): 691s + [Epoch 164][40/66] loss=0.849232 avg=0.852973 VRAM=38.6GiB | 54.9% done | ETA(epoch): 499s + [Epoch 164][50/66] loss=0.952110 avg=0.850365 VRAM=38.7GiB | 54.9% done | ETA(epoch): 307s + [Epoch 164][60/66] loss=0.930817 avg=0.844714 VRAM=38.6GiB | 55.0% done | ETA(epoch): 115s + Train loss: 0.849360 (1266.3s) ETA: 3156min + Val loss: 0.901186 [t_0.0-0.2=1.0864 t_0.2-0.4=0.9950 t_0.4-0.6=0.8466 t_0.6-0.8=0.7327 t_0.8-1.0=0.6397] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 165/299 (55% done) --- + [Epoch 165][10/66] loss=0.938314 avg=0.844095 VRAM=38.7GiB | 55.1% done | ETA(epoch): 1075s + [Epoch 165][20/66] loss=0.792689 avg=0.840996 VRAM=38.6GiB | 55.1% done | ETA(epoch): 883s + [Epoch 165][30/66] loss=0.894735 avg=0.855724 VRAM=38.7GiB | 55.2% done | ETA(epoch): 691s + [Epoch 165][40/66] loss=0.896473 avg=0.857527 VRAM=38.6GiB | 55.2% done | ETA(epoch): 499s + [Epoch 165][50/66] loss=0.863485 avg=0.854482 VRAM=38.7GiB | 55.3% done | ETA(epoch): 307s + [Epoch 165][60/66] loss=0.725696 avg=0.852863 VRAM=38.6GiB | 55.3% done | ETA(epoch): 115s + Train loss: 0.855900 (1267.7s) ETA: 3132min + Val loss: 0.844257 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9891 t_0.4-0.6=0.8633 t_0.6-0.8=0.7097 t_0.8-1.0=0.6318] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 166/299 (55% done) --- + [Epoch 166][10/66] loss=0.781184 avg=0.854040 VRAM=38.7GiB | 55.4% done | ETA(epoch): 1074s + [Epoch 166][20/66] loss=0.814251 avg=0.855667 VRAM=38.6GiB | 55.4% done | ETA(epoch): 883s + [Epoch 166][30/66] loss=0.887125 avg=0.869100 VRAM=38.7GiB | 55.5% done | ETA(epoch): 691s + [Epoch 166][40/66] loss=0.927168 avg=0.860162 VRAM=38.6GiB | 55.5% done | ETA(epoch): 499s + [Epoch 166][50/66] loss=0.818964 avg=0.856962 VRAM=38.7GiB | 55.6% done | ETA(epoch): 307s + [Epoch 166][60/66] loss=0.820859 avg=0.851519 VRAM=38.6GiB | 55.6% done | ETA(epoch): 115s + Train loss: 0.850119 (1267.9s) ETA: 3108min + Val loss: 0.909589 [t_0.0-0.2=1.0768 t_0.2-0.4=1.0003 t_0.4-0.6=0.8606 t_0.6-0.8=0.7148 t_0.8-1.0=0.6526] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 167/299 (56% done) --- + [Epoch 167][10/66] loss=0.847853 avg=0.852278 VRAM=38.7GiB | 55.7% done | ETA(epoch): 1075s + [Epoch 167][20/66] loss=0.784940 avg=0.850638 VRAM=38.6GiB | 55.8% done | ETA(epoch): 884s + [Epoch 167][30/66] loss=0.936194 avg=0.850415 VRAM=38.7GiB | 55.8% done | ETA(epoch): 692s + [Epoch 167][40/66] loss=0.769695 avg=0.836589 VRAM=38.6GiB | 55.9% done | ETA(epoch): 499s + [Epoch 167][50/66] loss=0.871939 avg=0.838330 VRAM=38.7GiB | 55.9% done | ETA(epoch): 307s + [Epoch 167][60/66] loss=0.902232 avg=0.843628 VRAM=38.6GiB | 56.0% done | ETA(epoch): 115s + Train loss: 0.840718 (1268.3s) ETA: 3084min + Val loss: 0.871045 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0089 t_0.4-0.6=0.8377 t_0.6-0.8=0.6967 t_0.8-1.0=0.6389] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 168/299 (56% done) --- + [Epoch 168][10/66] loss=0.845669 avg=0.841462 VRAM=38.7GiB | 56.1% done | ETA(epoch): 1074s + [Epoch 168][20/66] loss=0.847199 avg=0.851949 VRAM=38.6GiB | 56.1% done | ETA(epoch): 883s + [Epoch 168][30/66] loss=0.721718 avg=0.842653 VRAM=38.7GiB | 56.2% done | ETA(epoch): 692s + [Epoch 168][40/66] loss=0.811220 avg=0.848490 VRAM=38.6GiB | 56.2% done | ETA(epoch): 502s + [Epoch 168][50/66] loss=0.731240 avg=0.847832 VRAM=38.7GiB | 56.3% done | ETA(epoch): 308s + [Epoch 168][60/66] loss=0.795757 avg=0.847001 VRAM=38.6GiB | 56.3% done | ETA(epoch): 116s + Train loss: 0.847806 (1271.1s) ETA: 3060min + Val loss: 0.824798 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9865 t_0.4-0.6=0.8547 t_0.6-0.8=0.6899 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 169/299 (56% done) --- + [Epoch 169][10/66] loss=0.895131 avg=0.863259 VRAM=38.7GiB | 56.4% done | ETA(epoch): 1076s + [Epoch 169][20/66] loss=0.813514 avg=0.855207 VRAM=38.6GiB | 56.4% done | ETA(epoch): 884s + [Epoch 169][30/66] loss=0.849147 avg=0.863969 VRAM=38.7GiB | 56.5% done | ETA(epoch): 692s + [Epoch 169][40/66] loss=0.853686 avg=0.865387 VRAM=38.6GiB | 56.5% done | ETA(epoch): 500s + [Epoch 169][50/66] loss=0.914664 avg=0.867100 VRAM=38.7GiB | 56.6% done | ETA(epoch): 307s + [Epoch 169][60/66] loss=0.909554 avg=0.868990 VRAM=38.6GiB | 56.6% done | ETA(epoch): 115s + Train loss: 0.862983 (1267.9s) ETA: 3036min + Val loss: 0.831373 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9448 t_0.4-0.6=0.8395 t_0.6-0.8=0.7210 t_0.8-1.0=0.6357] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 170/299 (57% done) --- + [Epoch 170][10/66] loss=0.868485 avg=0.867497 VRAM=38.7GiB | 56.7% done | ETA(epoch): 1075s + [Epoch 170][20/66] loss=0.965297 avg=0.856231 VRAM=38.6GiB | 56.8% done | ETA(epoch): 883s + [Epoch 170][30/66] loss=0.914131 avg=0.857622 VRAM=38.7GiB | 56.8% done | ETA(epoch): 691s + [Epoch 170][40/66] loss=0.913562 avg=0.866552 VRAM=38.6GiB | 56.9% done | ETA(epoch): 499s + [Epoch 170][50/66] loss=0.874925 avg=0.865414 VRAM=38.7GiB | 56.9% done | ETA(epoch): 307s + [Epoch 170][60/66] loss=0.815256 avg=0.863120 VRAM=38.6GiB | 57.0% done | ETA(epoch): 115s + Train loss: 0.861924 (1266.8s) ETA: 3012min + Val loss: 0.870538 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9979 t_0.4-0.6=0.8017 t_0.6-0.8=0.7367 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 171/299 (57% done) --- + [Epoch 171][10/66] loss=0.932432 avg=0.846283 VRAM=38.7GiB | 57.1% done | ETA(epoch): 1086s + [Epoch 171][20/66] loss=0.922739 avg=0.844425 VRAM=38.6GiB | 57.1% done | ETA(epoch): 888s + [Epoch 171][30/66] loss=0.946445 avg=0.853710 VRAM=38.7GiB | 57.2% done | ETA(epoch): 693s + [Epoch 171][40/66] loss=0.781640 avg=0.850943 VRAM=38.6GiB | 57.2% done | ETA(epoch): 501s + [Epoch 171][50/66] loss=0.972077 avg=0.858298 VRAM=38.7GiB | 57.3% done | ETA(epoch): 308s + [Epoch 171][60/66] loss=0.911692 avg=0.857059 VRAM=38.6GiB | 57.3% done | ETA(epoch): 115s + Train loss: 0.859710 (1269.3s) ETA: 2989min + Val loss: 0.871369 [t_0.0-0.2=1.0694 t_0.2-0.4=1.0043 t_0.4-0.6=0.8747 t_0.6-0.8=0.7134 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 20.8/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 172/299 (57% done) --- + [Epoch 172][10/66] loss=0.762637 avg=0.888015 VRAM=38.7GiB | 57.4% done | ETA(epoch): 1073s + [Epoch 172][20/66] loss=0.796476 avg=0.876902 VRAM=38.6GiB | 57.4% done | ETA(epoch): 882s + [Epoch 172][30/66] loss=0.794500 avg=0.865711 VRAM=38.7GiB | 57.5% done | ETA(epoch): 691s + [Epoch 172][40/66] loss=0.758065 avg=0.865731 VRAM=38.6GiB | 57.5% done | ETA(epoch): 499s + [Epoch 172][50/66] loss=0.923756 avg=0.861334 VRAM=38.7GiB | 57.6% done | ETA(epoch): 307s + [Epoch 172][60/66] loss=0.908272 avg=0.860762 VRAM=38.6GiB | 57.6% done | ETA(epoch): 115s + Train loss: 0.862474 (1267.7s) ETA: 2965min + Val loss: 0.896693 [t_0.0-0.2=1.0605 t_0.2-0.4=1.0064 t_0.4-0.6=0.8845 t_0.6-0.8=0.7320 t_0.8-1.0=0.6497] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 173/299 (58% done) --- + [Epoch 173][10/66] loss=0.905197 avg=0.849841 VRAM=38.7GiB | 57.7% done | ETA(epoch): 1074s + [Epoch 173][20/66] loss=0.939411 avg=0.851971 VRAM=38.6GiB | 57.8% done | ETA(epoch): 884s + [Epoch 173][30/66] loss=0.789096 avg=0.853505 VRAM=38.7GiB | 57.8% done | ETA(epoch): 691s + [Epoch 173][40/66] loss=0.878234 avg=0.856683 VRAM=38.6GiB | 57.9% done | ETA(epoch): 499s + [Epoch 173][50/66] loss=0.973917 avg=0.861137 VRAM=38.7GiB | 57.9% done | ETA(epoch): 307s + [Epoch 173][60/66] loss=0.840749 avg=0.862553 VRAM=38.6GiB | 58.0% done | ETA(epoch): 115s + Train loss: 0.860459 (1269.4s) ETA: 2941min + Val loss: 0.871129 [t_0.0-0.2=1.0665 t_0.2-0.4=1.0061 t_0.4-0.6=0.8687 t_0.6-0.8=0.7187 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 174/299 (58% done) --- + [Epoch 174][10/66] loss=0.811518 avg=0.841816 VRAM=38.7GiB | 58.1% done | ETA(epoch): 1076s + [Epoch 174][20/66] loss=0.865572 avg=0.855995 VRAM=38.6GiB | 58.1% done | ETA(epoch): 884s + [Epoch 174][30/66] loss=0.924726 avg=0.871025 VRAM=38.7GiB | 58.2% done | ETA(epoch): 691s + [Epoch 174][40/66] loss=0.969602 avg=0.882500 VRAM=38.6GiB | 58.2% done | ETA(epoch): 499s + [Epoch 174][50/66] loss=0.872055 avg=0.875665 VRAM=38.7GiB | 58.3% done | ETA(epoch): 309s + [Epoch 174][60/66] loss=0.724013 avg=0.867688 VRAM=38.6GiB | 58.3% done | ETA(epoch): 116s + Train loss: 0.861094 (1271.7s) ETA: 2917min + Val loss: 0.848541 [t_0.0-0.2=1.0842 t_0.2-0.4=1.0000 t_0.4-0.6=0.8633 t_0.6-0.8=0.6976 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 175/299 (58% done) --- + [Epoch 175][10/66] loss=0.876824 avg=0.893582 VRAM=38.7GiB | 58.4% done | ETA(epoch): 1075s + [Epoch 175][20/66] loss=0.970693 avg=0.884827 VRAM=38.6GiB | 58.4% done | ETA(epoch): 883s + [Epoch 175][30/66] loss=0.862311 avg=0.882548 VRAM=38.7GiB | 58.5% done | ETA(epoch): 691s + [Epoch 175][40/66] loss=0.797931 avg=0.874688 VRAM=38.6GiB | 58.5% done | ETA(epoch): 502s + [Epoch 175][50/66] loss=0.886536 avg=0.877562 VRAM=38.7GiB | 58.6% done | ETA(epoch): 309s + [Epoch 175][60/66] loss=0.916695 avg=0.874536 VRAM=38.6GiB | 58.6% done | ETA(epoch): 116s + Train loss: 0.875792 (1271.2s) ETA: 2893min + Val loss: 0.840560 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9910 t_0.4-0.6=0.8590 t_0.6-0.8=0.7086 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 176/299 (59% done) --- + [Epoch 176][10/66] loss=0.903984 avg=0.838049 VRAM=38.7GiB | 58.7% done | ETA(epoch): 1074s + [Epoch 176][20/66] loss=0.885558 avg=0.873640 VRAM=38.6GiB | 58.8% done | ETA(epoch): 882s + [Epoch 176][30/66] loss=0.761692 avg=0.864262 VRAM=38.7GiB | 58.8% done | ETA(epoch): 691s + [Epoch 176][40/66] loss=0.794016 avg=0.848629 VRAM=38.6GiB | 58.9% done | ETA(epoch): 501s + [Epoch 176][50/66] loss=0.839923 avg=0.851985 VRAM=38.7GiB | 58.9% done | ETA(epoch): 308s + [Epoch 176][60/66] loss=0.862009 avg=0.856029 VRAM=38.6GiB | 59.0% done | ETA(epoch): 115s + Train loss: 0.854580 (1269.1s) ETA: 2870min + Val loss: 0.860557 [t_0.0-0.2=1.0846 t_0.2-0.4=1.0002 t_0.4-0.6=0.8636 t_0.6-0.8=0.7165 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 20.9/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 177/299 (59% done) --- + [Epoch 177][10/66] loss=0.853990 avg=0.829224 VRAM=38.7GiB | 59.1% done | ETA(epoch): 1075s + [Epoch 177][20/66] loss=0.818801 avg=0.813782 VRAM=38.6GiB | 59.1% done | ETA(epoch): 883s + [Epoch 177][30/66] loss=0.575452 avg=0.830054 VRAM=38.7GiB | 59.2% done | ETA(epoch): 700s + [Epoch 177][40/66] loss=0.769902 avg=0.843667 VRAM=38.6GiB | 59.2% done | ETA(epoch): 504s + [Epoch 177][50/66] loss=0.788871 avg=0.843223 VRAM=38.7GiB | 59.3% done | ETA(epoch): 310s + [Epoch 177][60/66] loss=0.946991 avg=0.842095 VRAM=38.6GiB | 59.3% done | ETA(epoch): 116s + Train loss: 0.843985 (1274.7s) ETA: 2846min + Val loss: 0.879939 [t_0.0-0.2=1.0689 t_0.2-0.4=1.0040 t_0.4-0.6=0.8396 t_0.6-0.8=0.7141 t_0.8-1.0=0.6555] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 20.7/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 178/299 (59% done) --- + [Epoch 178][10/66] loss=0.897687 avg=0.834032 VRAM=38.7GiB | 59.4% done | ETA(epoch): 1077s + [Epoch 178][20/66] loss=0.817502 avg=0.816259 VRAM=38.6GiB | 59.4% done | ETA(epoch): 883s + [Epoch 178][30/66] loss=0.898349 avg=0.829833 VRAM=38.7GiB | 59.5% done | ETA(epoch): 691s + [Epoch 178][40/66] loss=0.748158 avg=0.832332 VRAM=38.6GiB | 59.5% done | ETA(epoch): 499s + [Epoch 178][50/66] loss=0.842442 avg=0.837976 VRAM=38.7GiB | 59.6% done | ETA(epoch): 307s + [Epoch 178][60/66] loss=0.804461 avg=0.837326 VRAM=38.6GiB | 59.6% done | ETA(epoch): 115s + Train loss: 0.841885 (1267.1s) ETA: 2822min + Val loss: 0.900303 [t_0.0-0.2=1.0641 t_0.2-0.4=0.9852 t_0.4-0.6=0.8558 t_0.6-0.8=0.7203 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 20.8/188.5 GiB (11.0%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 179/299 (60% done) --- + [Epoch 179][10/66] loss=0.804972 avg=0.861195 VRAM=38.7GiB | 59.7% done | ETA(epoch): 1075s + [Epoch 179][20/66] loss=0.783153 avg=0.849156 VRAM=38.6GiB | 59.8% done | ETA(epoch): 883s + [Epoch 179][30/66] loss=0.794986 avg=0.854249 VRAM=38.7GiB | 59.8% done | ETA(epoch): 691s + [Epoch 179][40/66] loss=0.826932 avg=0.850726 VRAM=38.6GiB | 59.9% done | ETA(epoch): 499s + [Epoch 179][50/66] loss=0.953967 avg=0.852026 VRAM=38.7GiB | 59.9% done | ETA(epoch): 307s + [Epoch 179][60/66] loss=0.829885 avg=0.856682 VRAM=38.6GiB | 60.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 11879 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 11879 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 11879 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 11879 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 11879 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 11879 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 11879 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 11879 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 11879 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 11879 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.855631 (2304.6s) ETA: 2810min + Val loss: 0.842354 [t_0.0-0.2=1.0773 t_0.2-0.4=1.0111 t_0.4-0.6=0.8336 t_0.6-0.8=0.7273 t_0.8-1.0=0.6566] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 180/299 (60% done) --- + [Epoch 180][10/66] loss=0.738031 avg=0.841270 VRAM=38.7GiB | 60.1% done | ETA(epoch): 1103s + [Epoch 180][20/66] loss=0.767196 avg=0.861787 VRAM=38.6GiB | 60.1% done | ETA(epoch): 894s + [Epoch 180][30/66] loss=0.812627 avg=0.850091 VRAM=38.7GiB | 60.2% done | ETA(epoch): 697s + [Epoch 180][40/66] loss=0.835573 avg=0.843584 VRAM=38.6GiB | 60.2% done | ETA(epoch): 502s + [Epoch 180][50/66] loss=0.941306 avg=0.848637 VRAM=38.7GiB | 60.3% done | ETA(epoch): 309s + [Epoch 180][60/66] loss=0.899831 avg=0.849474 VRAM=38.6GiB | 60.3% done | ETA(epoch): 116s + Train loss: 0.847414 (1272.4s) ETA: 2786min + Val loss: 0.856684 [t_0.0-0.2=1.0597 t_0.2-0.4=1.0164 t_0.4-0.6=0.8296 t_0.6-0.8=0.7167 t_0.8-1.0=0.6551] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 181/299 (60% done) --- + [Epoch 181][10/66] loss=0.743702 avg=0.844273 VRAM=38.7GiB | 60.4% done | ETA(epoch): 1075s + [Epoch 181][20/66] loss=0.802168 avg=0.836239 VRAM=38.6GiB | 60.4% done | ETA(epoch): 883s + [Epoch 181][30/66] loss=0.746059 avg=0.860491 VRAM=38.7GiB | 60.5% done | ETA(epoch): 692s + [Epoch 181][40/66] loss=0.771457 avg=0.855588 VRAM=38.6GiB | 60.5% done | ETA(epoch): 499s + [Epoch 181][50/66] loss=0.915526 avg=0.852422 VRAM=38.7GiB | 60.6% done | ETA(epoch): 307s + [Epoch 181][60/66] loss=0.831208 avg=0.847065 VRAM=38.6GiB | 60.6% done | ETA(epoch): 115s + Train loss: 0.845400 (1268.2s) ETA: 2762min + Val loss: 0.893162 [t_0.0-0.2=1.0772 t_0.2-0.4=0.9842 t_0.4-0.6=0.8434 t_0.6-0.8=0.7127 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 182/299 (61% done) --- + [Epoch 182][10/66] loss=0.787934 avg=0.849522 VRAM=38.7GiB | 60.7% done | ETA(epoch): 1075s + [Epoch 182][20/66] loss=0.915580 avg=0.857809 VRAM=38.6GiB | 60.8% done | ETA(epoch): 883s + [Epoch 182][30/66] loss=0.909457 avg=0.861915 VRAM=38.7GiB | 60.8% done | ETA(epoch): 691s + [Epoch 182][40/66] loss=0.880637 avg=0.862971 VRAM=38.6GiB | 60.9% done | ETA(epoch): 499s + [Epoch 182][50/66] loss=0.770424 avg=0.861000 VRAM=38.7GiB | 60.9% done | ETA(epoch): 307s + [Epoch 182][60/66] loss=0.785337 avg=0.856918 VRAM=38.6GiB | 61.0% done | ETA(epoch): 115s + Train loss: 0.860493 (1267.6s) ETA: 2738min + Val loss: 0.849583 [t_0.0-0.2=1.0597 t_0.2-0.4=0.9680 t_0.4-0.6=0.8489 t_0.6-0.8=0.7273 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 183/299 (61% done) --- + [Epoch 183][10/66] loss=0.944717 avg=0.852393 VRAM=38.7GiB | 61.1% done | ETA(epoch): 1074s + [Epoch 183][20/66] loss=0.944128 avg=0.854623 VRAM=38.6GiB | 61.1% done | ETA(epoch): 882s + [Epoch 183][30/66] loss=0.858541 avg=0.852903 VRAM=38.7GiB | 61.2% done | ETA(epoch): 691s + [Epoch 183][40/66] loss=0.986144 avg=0.855213 VRAM=38.6GiB | 61.2% done | ETA(epoch): 499s + [Epoch 183][50/66] loss=0.792809 avg=0.855579 VRAM=38.7GiB | 61.3% done | ETA(epoch): 307s + [Epoch 183][60/66] loss=0.939675 avg=0.849233 VRAM=38.6GiB | 61.3% done | ETA(epoch): 115s + Train loss: 0.851519 (1266.3s) ETA: 2715min + Val loss: 0.823383 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9885 t_0.4-0.6=0.8459 t_0.6-0.8=0.6961 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 184/299 (61% done) --- + [Epoch 184][10/66] loss=0.858853 avg=0.869888 VRAM=38.7GiB | 61.4% done | ETA(epoch): 1077s + [Epoch 184][20/66] loss=0.921332 avg=0.879878 VRAM=38.6GiB | 61.4% done | ETA(epoch): 884s + [Epoch 184][30/66] loss=0.882668 avg=0.876018 VRAM=38.7GiB | 61.5% done | ETA(epoch): 691s + [Epoch 184][40/66] loss=0.834967 avg=0.866757 VRAM=38.6GiB | 61.5% done | ETA(epoch): 500s + [Epoch 184][50/66] loss=0.907827 avg=0.868695 VRAM=38.7GiB | 61.6% done | ETA(epoch): 307s + [Epoch 184][60/66] loss=0.815177 avg=0.866909 VRAM=38.6GiB | 61.6% done | ETA(epoch): 115s + Train loss: 0.866931 (1267.7s) ETA: 2691min + Val loss: 0.845956 [t_0.0-0.2=1.0777 t_0.2-0.4=0.9786 t_0.4-0.6=0.8593 t_0.6-0.8=0.6930 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 185/299 (62% done) --- + [Epoch 185][10/66] loss=0.982956 avg=0.861916 VRAM=38.7GiB | 61.7% done | ETA(epoch): 1073s + [Epoch 185][20/66] loss=0.754875 avg=0.848838 VRAM=38.6GiB | 61.8% done | ETA(epoch): 882s + [Epoch 185][30/66] loss=0.928185 avg=0.854290 VRAM=38.7GiB | 61.8% done | ETA(epoch): 695s + [Epoch 185][40/66] loss=0.797332 avg=0.851200 VRAM=38.6GiB | 61.9% done | ETA(epoch): 501s + [Epoch 185][50/66] loss=0.971149 avg=0.854027 VRAM=38.7GiB | 61.9% done | ETA(epoch): 308s + [Epoch 185][60/66] loss=0.815317 avg=0.854108 VRAM=38.6GiB | 62.0% done | ETA(epoch): 116s + Train loss: 0.863098 (1270.6s) ETA: 2667min + Val loss: 0.827614 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0045 t_0.4-0.6=0.8392 t_0.6-0.8=0.7122 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 186/299 (62% done) --- + [Epoch 186][10/66] loss=0.821700 avg=0.849597 VRAM=38.7GiB | 62.1% done | ETA(epoch): 1074s + [Epoch 186][20/66] loss=0.837815 avg=0.839136 VRAM=38.6GiB | 62.1% done | ETA(epoch): 883s + [Epoch 186][30/66] loss=0.775850 avg=0.831404 VRAM=38.7GiB | 62.2% done | ETA(epoch): 691s + [Epoch 186][40/66] loss=0.898412 avg=0.839812 VRAM=38.6GiB | 62.2% done | ETA(epoch): 499s + [Epoch 186][50/66] loss=0.745044 avg=0.840062 VRAM=38.7GiB | 62.3% done | ETA(epoch): 307s + [Epoch 186][60/66] loss=0.807987 avg=0.840166 VRAM=38.6GiB | 62.3% done | ETA(epoch): 115s + Train loss: 0.842886 (1267.1s) ETA: 2643min + Val loss: 0.823364 [t_0.0-0.2=1.0680 t_0.2-0.4=1.0259 t_0.4-0.6=0.8316 t_0.6-0.8=0.7190 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 187/299 (62% done) --- + [Epoch 187][10/66] loss=0.922393 avg=0.835127 VRAM=38.7GiB | 62.4% done | ETA(epoch): 1075s + [Epoch 187][20/66] loss=0.741876 avg=0.833195 VRAM=38.6GiB | 62.4% done | ETA(epoch): 882s + [Epoch 187][30/66] loss=0.936162 avg=0.839975 VRAM=38.7GiB | 62.5% done | ETA(epoch): 690s + [Epoch 187][40/66] loss=0.796265 avg=0.855509 VRAM=38.6GiB | 62.5% done | ETA(epoch): 499s + [Epoch 187][50/66] loss=1.019448 avg=0.863511 VRAM=38.7GiB | 62.6% done | ETA(epoch): 307s + [Epoch 187][60/66] loss=0.947645 avg=0.863459 VRAM=38.6GiB | 62.6% done | ETA(epoch): 115s + Train loss: 0.864954 (1266.7s) ETA: 2619min + Val loss: 0.882447 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0188 t_0.4-0.6=0.8623 t_0.6-0.8=0.7224 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 188/299 (63% done) --- + [Epoch 188][10/66] loss=0.727071 avg=0.895041 VRAM=38.7GiB | 62.7% done | ETA(epoch): 1075s + [Epoch 188][20/66] loss=0.913729 avg=0.902711 VRAM=38.6GiB | 62.8% done | ETA(epoch): 882s + [Epoch 188][30/66] loss=0.899100 avg=0.888175 VRAM=38.7GiB | 62.8% done | ETA(epoch): 691s + [Epoch 188][40/66] loss=0.779958 avg=0.884540 VRAM=38.6GiB | 62.9% done | ETA(epoch): 499s + [Epoch 188][50/66] loss=0.817509 avg=0.878868 VRAM=38.7GiB | 62.9% done | ETA(epoch): 307s + [Epoch 188][60/66] loss=0.936804 avg=0.876402 VRAM=38.6GiB | 63.0% done | ETA(epoch): 115s + Train loss: 0.872193 (1269.4s) ETA: 2596min + Val loss: 0.832153 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0026 t_0.4-0.6=0.8495 t_0.6-0.8=0.6918 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 189/299 (63% done) --- + [Epoch 189][10/66] loss=0.946883 avg=0.883663 VRAM=38.7GiB | 63.1% done | ETA(epoch): 1075s + [Epoch 189][20/66] loss=0.870860 avg=0.855942 VRAM=38.6GiB | 63.1% done | ETA(epoch): 883s + [Epoch 189][30/66] loss=0.901439 avg=0.856316 VRAM=38.7GiB | 63.2% done | ETA(epoch): 691s + [Epoch 189][40/66] loss=0.832783 avg=0.856430 VRAM=38.6GiB | 63.2% done | ETA(epoch): 499s + [Epoch 189][50/66] loss=0.737660 avg=0.852142 VRAM=38.7GiB | 63.3% done | ETA(epoch): 307s + [Epoch 189][60/66] loss=0.888408 avg=0.853829 VRAM=38.6GiB | 63.3% done | ETA(epoch): 115s + Train loss: 0.857908 (1268.2s) ETA: 2572min + Val loss: 0.866468 [t_0.0-0.2=1.0838 t_0.2-0.4=0.9547 t_0.4-0.6=0.8835 t_0.6-0.8=0.6838 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 190/299 (63% done) --- + [Epoch 190][10/66] loss=0.846465 avg=0.861610 VRAM=38.7GiB | 63.4% done | ETA(epoch): 1074s + [Epoch 190][20/66] loss=0.853072 avg=0.863745 VRAM=38.6GiB | 63.4% done | ETA(epoch): 883s + [Epoch 190][30/66] loss=0.860583 avg=0.855660 VRAM=38.7GiB | 63.5% done | ETA(epoch): 691s + [Epoch 190][40/66] loss=0.881006 avg=0.866322 VRAM=38.6GiB | 63.5% done | ETA(epoch): 499s + [Epoch 190][50/66] loss=0.954042 avg=0.856566 VRAM=38.7GiB | 63.6% done | ETA(epoch): 308s + [Epoch 190][60/66] loss=0.812627 avg=0.856437 VRAM=38.6GiB | 63.6% done | ETA(epoch): 115s + Train loss: 0.857803 (1268.7s) ETA: 2548min + Val loss: 0.873669 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0107 t_0.4-0.6=0.8325 t_0.6-0.8=0.7092 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 191/299 (64% done) --- + [Epoch 191][10/66] loss=0.784366 avg=0.861436 VRAM=38.7GiB | 63.7% done | ETA(epoch): 1075s + [Epoch 191][20/66] loss=0.934383 avg=0.861130 VRAM=38.6GiB | 63.8% done | ETA(epoch): 883s + [Epoch 191][30/66] loss=0.735237 avg=0.852043 VRAM=38.7GiB | 63.8% done | ETA(epoch): 691s + [Epoch 191][40/66] loss=0.763479 avg=0.853064 VRAM=38.6GiB | 63.9% done | ETA(epoch): 499s + [Epoch 191][50/66] loss=0.942147 avg=0.849328 VRAM=38.7GiB | 63.9% done | ETA(epoch): 307s + [Epoch 191][60/66] loss=0.940411 avg=0.850256 VRAM=38.6GiB | 64.0% done | ETA(epoch): 115s + Train loss: 0.853962 (1266.3s) ETA: 2524min + Val loss: 0.876436 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0075 t_0.4-0.6=0.8292 t_0.6-0.8=0.6918 t_0.8-1.0=0.6269] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0191 + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 192/299 (64% done) --- + [Epoch 192][10/66] loss=0.942936 avg=0.866908 VRAM=38.7GiB | 64.1% done | ETA(epoch): 1074s + [Epoch 192][20/66] loss=0.880847 avg=0.868224 VRAM=38.6GiB | 64.1% done | ETA(epoch): 882s + [Epoch 192][30/66] loss=0.800312 avg=0.873400 VRAM=38.7GiB | 64.2% done | ETA(epoch): 690s + [Epoch 192][40/66] loss=0.719106 avg=0.859878 VRAM=38.6GiB | 64.2% done | ETA(epoch): 500s + [Epoch 192][50/66] loss=0.973844 avg=0.858108 VRAM=38.7GiB | 64.3% done | ETA(epoch): 307s + [Epoch 192][60/66] loss=0.823095 avg=0.854967 VRAM=38.6GiB | 64.3% done | ETA(epoch): 115s + Train loss: 0.852279 (1267.9s) ETA: 2501min + Val loss: 0.853267 [t_0.0-0.2=1.0740 t_0.2-0.4=1.0055 t_0.4-0.6=0.8535 t_0.6-0.8=0.7025 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 193/299 (64% done) --- + [Epoch 193][10/66] loss=0.854035 avg=0.853213 VRAM=38.7GiB | 64.4% done | ETA(epoch): 1074s + [Epoch 193][20/66] loss=0.810210 avg=0.855237 VRAM=38.6GiB | 64.4% done | ETA(epoch): 883s + [Epoch 193][30/66] loss=0.729671 avg=0.852184 VRAM=38.7GiB | 64.5% done | ETA(epoch): 691s + [Epoch 193][40/66] loss=0.910457 avg=0.856402 VRAM=38.6GiB | 64.5% done | ETA(epoch): 499s + [Epoch 193][50/66] loss=0.941680 avg=0.857225 VRAM=38.7GiB | 64.6% done | ETA(epoch): 307s + [Epoch 193][60/66] loss=0.736962 avg=0.856243 VRAM=38.6GiB | 64.6% done | ETA(epoch): 115s + Train loss: 0.852434 (1266.0s) ETA: 2477min + Val loss: 0.839807 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9733 t_0.4-0.6=0.8272 t_0.6-0.8=0.7428 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 194/299 (65% done) --- + [Epoch 194][10/66] loss=0.857321 avg=0.838691 VRAM=38.7GiB | 64.7% done | ETA(epoch): 1072s + [Epoch 194][20/66] loss=0.871474 avg=0.838632 VRAM=38.6GiB | 64.8% done | ETA(epoch): 880s + [Epoch 194][30/66] loss=0.915295 avg=0.834713 VRAM=38.7GiB | 64.8% done | ETA(epoch): 689s + [Epoch 194][40/66] loss=0.869480 avg=0.852424 VRAM=38.6GiB | 64.9% done | ETA(epoch): 498s + [Epoch 194][50/66] loss=0.880450 avg=0.852692 VRAM=38.7GiB | 64.9% done | ETA(epoch): 306s + [Epoch 194][60/66] loss=0.766019 avg=0.847460 VRAM=38.6GiB | 65.0% done | ETA(epoch): 115s + Train loss: 0.848783 (1263.0s) ETA: 2453min + Val loss: 0.882964 [t_0.0-0.2=1.0704 t_0.2-0.4=0.9763 t_0.4-0.6=0.8813 t_0.6-0.8=0.7075 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 195/299 (65% done) --- + [Epoch 195][10/66] loss=0.976595 avg=0.865277 VRAM=38.7GiB | 65.1% done | ETA(epoch): 1090s + [Epoch 195][20/66] loss=0.797580 avg=0.848894 VRAM=38.6GiB | 65.1% done | ETA(epoch): 889s + [Epoch 195][30/66] loss=0.920743 avg=0.857703 VRAM=38.7GiB | 65.2% done | ETA(epoch): 694s + [Epoch 195][40/66] loss=0.910621 avg=0.856450 VRAM=38.6GiB | 65.2% done | ETA(epoch): 501s + [Epoch 195][50/66] loss=0.879854 avg=0.851373 VRAM=38.7GiB | 65.3% done | ETA(epoch): 308s + [Epoch 195][60/66] loss=0.850001 avg=0.852354 VRAM=38.6GiB | 65.3% done | ETA(epoch): 115s + Train loss: 0.852183 (1268.6s) ETA: 2429min + Val loss: 0.850121 [t_0.0-0.2=1.0796 t_0.2-0.4=1.0234 t_0.4-0.6=0.8563 t_0.6-0.8=0.6997 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 196/299 (65% done) --- + [Epoch 196][10/66] loss=0.817721 avg=0.856796 VRAM=38.7GiB | 65.4% done | ETA(epoch): 1091s + [Epoch 196][20/66] loss=0.835157 avg=0.868538 VRAM=38.6GiB | 65.4% done | ETA(epoch): 889s + [Epoch 196][30/66] loss=0.959156 avg=0.861078 VRAM=38.7GiB | 65.5% done | ETA(epoch): 694s + [Epoch 196][40/66] loss=0.792726 avg=0.852782 VRAM=38.6GiB | 65.5% done | ETA(epoch): 501s + [Epoch 196][50/66] loss=0.850513 avg=0.854736 VRAM=38.7GiB | 65.6% done | ETA(epoch): 308s + [Epoch 196][60/66] loss=0.924966 avg=0.857230 VRAM=38.6GiB | 65.6% done | ETA(epoch): 115s + Train loss: 0.854303 (1269.9s) ETA: 2406min + Val loss: 0.856628 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9950 t_0.4-0.6=0.8278 t_0.6-0.8=0.7271 t_0.8-1.0=0.6160] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 197/299 (66% done) --- + [Epoch 197][10/66] loss=0.850011 avg=0.856264 VRAM=38.7GiB | 65.7% done | ETA(epoch): 1074s + [Epoch 197][20/66] loss=0.765106 avg=0.849821 VRAM=38.6GiB | 65.8% done | ETA(epoch): 882s + [Epoch 197][30/66] loss=0.827844 avg=0.826369 VRAM=38.7GiB | 65.8% done | ETA(epoch): 691s + [Epoch 197][40/66] loss=0.888337 avg=0.833136 VRAM=38.6GiB | 65.9% done | ETA(epoch): 499s + [Epoch 197][50/66] loss=0.748352 avg=0.827800 VRAM=38.7GiB | 65.9% done | ETA(epoch): 307s + [Epoch 197][60/66] loss=0.848038 avg=0.835701 VRAM=38.6GiB | 66.0% done | ETA(epoch): 115s + Train loss: 0.833475 (1268.5s) ETA: 2382min + Val loss: 0.856617 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9716 t_0.4-0.6=0.8491 t_0.6-0.8=0.6902 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 198/299 (66% done) --- + [Epoch 198][10/66] loss=0.792013 avg=0.856878 VRAM=38.7GiB | 66.1% done | ETA(epoch): 1076s + [Epoch 198][20/66] loss=0.919245 avg=0.864002 VRAM=38.6GiB | 66.1% done | ETA(epoch): 883s + [Epoch 198][30/66] loss=0.797367 avg=0.863282 VRAM=38.7GiB | 66.2% done | ETA(epoch): 691s + [Epoch 198][40/66] loss=0.832343 avg=0.858324 VRAM=38.6GiB | 66.2% done | ETA(epoch): 499s + [Epoch 198][50/66] loss=0.899493 avg=0.861665 VRAM=38.7GiB | 66.3% done | ETA(epoch): 308s + [Epoch 198][60/66] loss=0.798831 avg=0.860998 VRAM=38.6GiB | 66.3% done | ETA(epoch): 115s + Train loss: 0.855963 (1268.1s) ETA: 2358min + Val loss: 0.852759 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9697 t_0.4-0.6=0.8746 t_0.6-0.8=0.6938 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 199/299 (66% done) --- + [Epoch 199][10/66] loss=0.870542 avg=0.818696 VRAM=38.7GiB | 66.4% done | ETA(epoch): 1073s + [Epoch 199][20/66] loss=0.879546 avg=0.828900 VRAM=38.6GiB | 66.4% done | ETA(epoch): 881s + [Epoch 199][30/66] loss=0.989457 avg=0.842081 VRAM=38.7GiB | 66.5% done | ETA(epoch): 690s + [Epoch 199][40/66] loss=0.987369 avg=0.853122 VRAM=38.6GiB | 66.5% done | ETA(epoch): 500s + [Epoch 199][50/66] loss=0.727128 avg=0.851678 VRAM=38.7GiB | 66.6% done | ETA(epoch): 308s + [Epoch 199][60/66] loss=0.978617 avg=0.854148 VRAM=38.6GiB | 66.6% done | ETA(epoch): 115s + Train loss: 0.851293 (1268.7s) ETA: 2335min + Val loss: 0.836835 [t_0.0-0.2=1.0741 t_0.2-0.4=0.9730 t_0.4-0.6=0.8325 t_0.6-0.8=0.7164 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 200/299 (67% done) --- + [Epoch 200][10/66] loss=0.940737 avg=0.867821 VRAM=38.7GiB | 66.7% done | ETA(epoch): 1074s + [Epoch 200][20/66] loss=0.757518 avg=0.853507 VRAM=38.6GiB | 66.8% done | ETA(epoch): 882s + [Epoch 200][30/66] loss=0.922006 avg=0.861147 VRAM=38.7GiB | 66.8% done | ETA(epoch): 690s + [Epoch 200][40/66] loss=0.837904 avg=0.859962 VRAM=38.6GiB | 66.9% done | ETA(epoch): 501s + [Epoch 200][50/66] loss=1.012298 avg=0.857599 VRAM=38.7GiB | 66.9% done | ETA(epoch): 308s + [Epoch 200][60/66] loss=0.943400 avg=0.860150 VRAM=38.6GiB | 67.0% done | ETA(epoch): 115s + Train loss: 0.860687 (1269.3s) ETA: 2311min + Val loss: 0.827709 [t_0.0-0.2=1.0542 t_0.2-0.4=1.0066 t_0.4-0.6=0.8142 t_0.6-0.8=0.7261 t_0.8-1.0=0.6577] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 21.7/188.5 GiB (11.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 201/299 (67% done) --- + [Epoch 201][10/66] loss=0.917679 avg=0.827348 VRAM=38.7GiB | 67.1% done | ETA(epoch): 1075s + [Epoch 201][20/66] loss=0.934503 avg=0.850754 VRAM=38.6GiB | 67.1% done | ETA(epoch): 882s + [Epoch 201][30/66] loss=0.826160 avg=0.846311 VRAM=38.7GiB | 67.2% done | ETA(epoch): 691s + [Epoch 201][40/66] loss=0.958904 avg=0.835664 VRAM=38.6GiB | 67.2% done | ETA(epoch): 499s + [Epoch 201][50/66] loss=0.966722 avg=0.841729 VRAM=38.7GiB | 67.3% done | ETA(epoch): 307s + [Epoch 201][60/66] loss=0.832263 avg=0.844027 VRAM=38.6GiB | 67.3% done | ETA(epoch): 115s + Train loss: 0.841237 (1265.9s) ETA: 2287min + Val loss: 0.881774 [t_0.0-0.2=1.0772 t_0.2-0.4=0.9506 t_0.4-0.6=0.8871 t_0.6-0.8=0.7135 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 202/299 (67% done) --- + [Epoch 202][10/66] loss=0.846713 avg=0.857609 VRAM=38.7GiB | 67.4% done | ETA(epoch): 1075s + [Epoch 202][20/66] loss=0.974806 avg=0.855362 VRAM=38.6GiB | 67.4% done | ETA(epoch): 891s + [Epoch 202][30/66] loss=0.827211 avg=0.862591 VRAM=38.7GiB | 67.5% done | ETA(epoch): 696s + [Epoch 202][40/66] loss=0.866772 avg=0.851337 VRAM=38.6GiB | 67.5% done | ETA(epoch): 502s + [Epoch 202][50/66] loss=0.783002 avg=0.847979 VRAM=38.7GiB | 67.6% done | ETA(epoch): 308s + [Epoch 202][60/66] loss=0.793469 avg=0.848029 VRAM=38.6GiB | 67.6% done | ETA(epoch): 116s + Train loss: 0.847008 (1270.2s) ETA: 2264min + Val loss: 0.846109 [t_0.0-0.2=1.0603 t_0.2-0.4=1.0119 t_0.4-0.6=0.8309 t_0.6-0.8=0.7037 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 203/299 (68% done) --- + [Epoch 203][10/66] loss=0.888561 avg=0.818305 VRAM=38.7GiB | 67.7% done | ETA(epoch): 1087s + [Epoch 203][20/66] loss=0.871690 avg=0.846786 VRAM=38.6GiB | 67.8% done | ETA(epoch): 888s + [Epoch 203][30/66] loss=1.017450 avg=0.851136 VRAM=38.7GiB | 67.8% done | ETA(epoch): 693s + [Epoch 203][40/66] loss=0.869884 avg=0.843817 VRAM=38.6GiB | 67.9% done | ETA(epoch): 500s + [Epoch 203][50/66] loss=0.897332 avg=0.850633 VRAM=38.7GiB | 67.9% done | ETA(epoch): 308s + [Epoch 203][60/66] loss=0.909750 avg=0.852391 VRAM=38.6GiB | 68.0% done | ETA(epoch): 115s + Train loss: 0.854593 (1269.2s) ETA: 2240min + Val loss: 0.842244 [t_0.0-0.2=1.0818 t_0.2-0.4=1.0111 t_0.4-0.6=0.8570 t_0.6-0.8=0.7207 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 204/299 (68% done) --- + [Epoch 204][10/66] loss=0.818883 avg=0.880218 VRAM=38.7GiB | 68.1% done | ETA(epoch): 1074s + [Epoch 204][20/66] loss=0.809494 avg=0.863166 VRAM=38.6GiB | 68.1% done | ETA(epoch): 883s + [Epoch 204][30/66] loss=0.941991 avg=0.850969 VRAM=38.7GiB | 68.2% done | ETA(epoch): 691s + [Epoch 204][40/66] loss=0.797612 avg=0.849464 VRAM=38.6GiB | 68.2% done | ETA(epoch): 499s + [Epoch 204][50/66] loss=0.900030 avg=0.844634 VRAM=38.7GiB | 68.3% done | ETA(epoch): 307s + [Epoch 204][60/66] loss=0.778445 avg=0.842480 VRAM=38.6GiB | 68.3% done | ETA(epoch): 115s + Train loss: 0.843773 (1270.1s) ETA: 2216min + Val loss: 0.856728 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9658 t_0.4-0.6=0.8522 t_0.6-0.8=0.7374 t_0.8-1.0=0.6553] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 205/299 (68% done) --- + [Epoch 205][10/66] loss=0.839635 avg=0.879388 VRAM=38.7GiB | 68.4% done | ETA(epoch): 1074s + [Epoch 205][20/66] loss=0.799830 avg=0.871826 VRAM=38.6GiB | 68.4% done | ETA(epoch): 882s + [Epoch 205][30/66] loss=0.806481 avg=0.868334 VRAM=38.7GiB | 68.5% done | ETA(epoch): 691s + [Epoch 205][40/66] loss=0.851120 avg=0.860774 VRAM=38.6GiB | 68.5% done | ETA(epoch): 499s + [Epoch 205][50/66] loss=0.957978 avg=0.857524 VRAM=38.7GiB | 68.6% done | ETA(epoch): 307s + [Epoch 205][60/66] loss=0.697318 avg=0.848465 VRAM=38.6GiB | 68.6% done | ETA(epoch): 115s + Train loss: 0.848581 (1269.5s) ETA: 2193min + Val loss: 0.827942 [t_0.0-0.2=1.0705 t_0.2-0.4=0.9979 t_0.4-0.6=0.8471 t_0.6-0.8=0.6941 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 206/299 (69% done) --- + [Epoch 206][10/66] loss=0.817351 avg=0.893958 VRAM=38.7GiB | 68.7% done | ETA(epoch): 1073s + [Epoch 206][20/66] loss=0.804770 avg=0.856757 VRAM=38.6GiB | 68.8% done | ETA(epoch): 882s + [Epoch 206][30/66] loss=0.823095 avg=0.858814 VRAM=38.7GiB | 68.8% done | ETA(epoch): 690s + [Epoch 206][40/66] loss=0.961591 avg=0.855422 VRAM=38.6GiB | 68.9% done | ETA(epoch): 499s + [Epoch 206][50/66] loss=0.811987 avg=0.849151 VRAM=38.7GiB | 68.9% done | ETA(epoch): 307s + [Epoch 206][60/66] loss=0.842024 avg=0.847396 VRAM=38.6GiB | 69.0% done | ETA(epoch): 115s + Train loss: 0.849076 (1268.4s) ETA: 2169min + Val loss: 0.848452 [t_0.0-0.2=1.0570 t_0.2-0.4=1.0004 t_0.4-0.6=0.8672 t_0.6-0.8=0.7093 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 207/299 (69% done) --- + [Epoch 207][10/66] loss=0.862700 avg=0.825904 VRAM=38.7GiB | 69.1% done | ETA(epoch): 1074s + [Epoch 207][20/66] loss=0.971308 avg=0.837192 VRAM=38.6GiB | 69.1% done | ETA(epoch): 882s + [Epoch 207][30/66] loss=0.763184 avg=0.843987 VRAM=38.7GiB | 69.2% done | ETA(epoch): 691s + [Epoch 207][40/66] loss=0.714846 avg=0.844906 VRAM=38.6GiB | 69.2% done | ETA(epoch): 499s + [Epoch 207][50/66] loss=0.951911 avg=0.847004 VRAM=38.7GiB | 69.3% done | ETA(epoch): 307s + [Epoch 207][60/66] loss=1.007057 avg=0.851416 VRAM=38.6GiB | 69.3% done | ETA(epoch): 115s + Train loss: 0.851455 (1266.2s) ETA: 2146min + Val loss: 0.836491 [t_0.0-0.2=1.0591 t_0.2-0.4=0.9784 t_0.4-0.6=0.8362 t_0.6-0.8=0.7328 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 208/299 (69% done) --- + [Epoch 208][10/66] loss=0.757496 avg=0.803645 VRAM=38.7GiB | 69.4% done | ETA(epoch): 1073s + [Epoch 208][20/66] loss=0.805242 avg=0.836220 VRAM=38.6GiB | 69.4% done | ETA(epoch): 882s + [Epoch 208][30/66] loss=0.890335 avg=0.836972 VRAM=38.7GiB | 69.5% done | ETA(epoch): 690s + [Epoch 208][40/66] loss=0.870397 avg=0.841720 VRAM=38.6GiB | 69.5% done | ETA(epoch): 499s + [Epoch 208][50/66] loss=0.827336 avg=0.847592 VRAM=38.7GiB | 69.6% done | ETA(epoch): 307s + [Epoch 208][60/66] loss=0.862135 avg=0.849299 VRAM=38.6GiB | 69.6% done | ETA(epoch): 115s + Train loss: 0.848579 (1266.3s) ETA: 2122min + Val loss: 0.863054 [t_0.0-0.2=1.0827 t_0.2-0.4=0.9933 t_0.4-0.6=0.8287 t_0.6-0.8=0.6960 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 209/299 (70% done) --- + [Epoch 209][10/66] loss=0.773383 avg=0.855953 VRAM=38.7GiB | 69.7% done | ETA(epoch): 1073s + [Epoch 209][20/66] loss=0.945947 avg=0.860833 VRAM=38.6GiB | 69.8% done | ETA(epoch): 883s + [Epoch 209][30/66] loss=0.937342 avg=0.854641 VRAM=38.7GiB | 69.8% done | ETA(epoch): 693s + [Epoch 209][40/66] loss=0.881077 avg=0.854668 VRAM=38.6GiB | 69.9% done | ETA(epoch): 500s + [Epoch 209][50/66] loss=0.855349 avg=0.854865 VRAM=38.7GiB | 69.9% done | ETA(epoch): 308s + [Epoch 209][60/66] loss=0.974665 avg=0.857801 VRAM=38.6GiB | 70.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 13859 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 13859 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 13859 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 13859 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 13859 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 13859 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 13859 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 13859 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 13859 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 13859 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.855163 (2303.6s) ETA: 2106min + Val loss: 0.802968 [t_0.0-0.2=1.0549 t_0.2-0.4=0.9751 t_0.4-0.6=0.8498 t_0.6-0.8=0.7158 t_0.8-1.0=0.6357] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0209 (BEST) + Deleted old checkpoint: checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 210/299 (70% done) --- + [Epoch 210][10/66] loss=0.723461 avg=0.811471 VRAM=38.7GiB | 70.1% done | ETA(epoch): 1075s + [Epoch 210][20/66] loss=0.879192 avg=0.839162 VRAM=38.6GiB | 70.1% done | ETA(epoch): 883s + [Epoch 210][30/66] loss=0.803020 avg=0.857135 VRAM=38.7GiB | 70.2% done | ETA(epoch): 691s + [Epoch 210][40/66] loss=0.689816 avg=0.856966 VRAM=38.6GiB | 70.2% done | ETA(epoch): 499s + [Epoch 210][50/66] loss=0.863586 avg=0.854138 VRAM=38.7GiB | 70.3% done | ETA(epoch): 307s + [Epoch 210][60/66] loss=0.926042 avg=0.855275 VRAM=38.6GiB | 70.3% done | ETA(epoch): 115s + Train loss: 0.853658 (1267.1s) ETA: 2082min + Val loss: 0.886654 [t_0.0-0.2=1.0728 t_0.2-0.4=1.0028 t_0.4-0.6=0.8743 t_0.6-0.8=0.7188 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 211/299 (70% done) --- + [Epoch 211][10/66] loss=0.863975 avg=0.852634 VRAM=38.7GiB | 70.4% done | ETA(epoch): 1075s + [Epoch 211][20/66] loss=0.839550 avg=0.872528 VRAM=38.6GiB | 70.4% done | ETA(epoch): 888s + [Epoch 211][30/66] loss=0.839218 avg=0.865268 VRAM=38.7GiB | 70.5% done | ETA(epoch): 694s + [Epoch 211][40/66] loss=0.963507 avg=0.867270 VRAM=38.6GiB | 70.5% done | ETA(epoch): 501s + [Epoch 211][50/66] loss=0.805740 avg=0.858344 VRAM=38.7GiB | 70.6% done | ETA(epoch): 308s + [Epoch 211][60/66] loss=0.834804 avg=0.857171 VRAM=38.6GiB | 70.6% done | ETA(epoch): 115s + Train loss: 0.854999 (1269.4s) ETA: 2058min + Val loss: 0.830899 [t_0.0-0.2=1.0860 t_0.2-0.4=0.9993 t_0.4-0.6=0.8505 t_0.6-0.8=0.7127 t_0.8-1.0=0.6346] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 212/299 (71% done) --- + [Epoch 212][10/66] loss=0.841916 avg=0.875280 VRAM=38.7GiB | 70.7% done | ETA(epoch): 1087s + [Epoch 212][20/66] loss=0.754300 avg=0.855688 VRAM=38.6GiB | 70.8% done | ETA(epoch): 888s + [Epoch 212][30/66] loss=0.885814 avg=0.858668 VRAM=38.7GiB | 70.8% done | ETA(epoch): 694s + [Epoch 212][40/66] loss=0.940682 avg=0.860434 VRAM=38.6GiB | 70.9% done | ETA(epoch): 501s + [Epoch 212][50/66] loss=0.902568 avg=0.859656 VRAM=38.7GiB | 70.9% done | ETA(epoch): 308s + [Epoch 212][60/66] loss=0.893266 avg=0.860161 VRAM=38.6GiB | 71.0% done | ETA(epoch): 115s + Train loss: 0.862692 (1269.9s) ETA: 2035min + Val loss: 0.817742 [t_0.0-0.2=1.0655 t_0.2-0.4=1.0114 t_0.4-0.6=0.8312 t_0.6-0.8=0.7227 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0212 +[MEM @ epoch 212 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 213/299 (71% done) --- + [Epoch 213][10/66] loss=0.904491 avg=0.874239 VRAM=38.7GiB | 71.1% done | ETA(epoch): 1075s + [Epoch 213][20/66] loss=0.877058 avg=0.873949 VRAM=38.6GiB | 71.1% done | ETA(epoch): 885s + [Epoch 213][30/66] loss=0.833937 avg=0.852201 VRAM=38.7GiB | 71.2% done | ETA(epoch): 692s + [Epoch 213][40/66] loss=0.954039 avg=0.859479 VRAM=38.6GiB | 71.2% done | ETA(epoch): 500s + [Epoch 213][50/66] loss=0.962160 avg=0.849794 VRAM=38.7GiB | 71.3% done | ETA(epoch): 307s + [Epoch 213][60/66] loss=0.794681 avg=0.852278 VRAM=38.6GiB | 71.3% done | ETA(epoch): 116s + Train loss: 0.852179 (1270.4s) ETA: 2011min + Val loss: 0.856035 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9478 t_0.4-0.6=0.8619 t_0.6-0.8=0.7251 t_0.8-1.0=0.6116] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 20.8/188.5 GiB (11.1%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 214/299 (71% done) --- + [Epoch 214][10/66] loss=0.914242 avg=0.851019 VRAM=38.7GiB | 71.4% done | ETA(epoch): 1074s + [Epoch 214][20/66] loss=0.783427 avg=0.846146 VRAM=38.6GiB | 71.4% done | ETA(epoch): 883s + [Epoch 214][30/66] loss=0.816677 avg=0.846374 VRAM=38.7GiB | 71.5% done | ETA(epoch): 691s + [Epoch 214][40/66] loss=0.977450 avg=0.843271 VRAM=38.6GiB | 71.5% done | ETA(epoch): 499s + [Epoch 214][50/66] loss=0.946628 avg=0.846449 VRAM=38.7GiB | 71.6% done | ETA(epoch): 307s + [Epoch 214][60/66] loss=0.875786 avg=0.850071 VRAM=38.6GiB | 71.6% done | ETA(epoch): 115s + Train loss: 0.849905 (1269.3s) ETA: 1988min + Val loss: 0.839733 [t_0.0-0.2=1.0616 t_0.2-0.4=1.0032 t_0.4-0.6=0.8419 t_0.6-0.8=0.7084 t_0.8-1.0=0.6441] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 215/299 (72% done) --- + [Epoch 215][10/66] loss=0.876003 avg=0.853551 VRAM=38.7GiB | 71.7% done | ETA(epoch): 1074s + [Epoch 215][20/66] loss=0.968427 avg=0.838325 VRAM=38.6GiB | 71.8% done | ETA(epoch): 883s + [Epoch 215][30/66] loss=0.951936 avg=0.836679 VRAM=38.7GiB | 71.8% done | ETA(epoch): 691s + [Epoch 215][40/66] loss=0.764468 avg=0.836808 VRAM=38.6GiB | 71.9% done | ETA(epoch): 499s + [Epoch 215][50/66] loss=0.880802 avg=0.833441 VRAM=38.7GiB | 71.9% done | ETA(epoch): 308s + [Epoch 215][60/66] loss=0.866689 avg=0.827644 VRAM=38.6GiB | 72.0% done | ETA(epoch): 115s + Train loss: 0.827860 (1269.5s) ETA: 1964min + Val loss: 0.854590 [t_0.0-0.2=1.0850 t_0.2-0.4=0.9785 t_0.4-0.6=0.8255 t_0.6-0.8=0.7304 t_0.8-1.0=0.6317] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 216/299 (72% done) --- + [Epoch 216][10/66] loss=0.891238 avg=0.857927 VRAM=38.7GiB | 72.1% done | ETA(epoch): 1077s + [Epoch 216][20/66] loss=0.802194 avg=0.856782 VRAM=38.6GiB | 72.1% done | ETA(epoch): 884s + [Epoch 216][30/66] loss=0.804121 avg=0.858314 VRAM=38.7GiB | 72.2% done | ETA(epoch): 692s + [Epoch 216][40/66] loss=0.928009 avg=0.853790 VRAM=38.6GiB | 72.2% done | ETA(epoch): 500s + [Epoch 216][50/66] loss=0.910845 avg=0.860744 VRAM=38.7GiB | 72.3% done | ETA(epoch): 308s + [Epoch 216][60/66] loss=0.846038 avg=0.863004 VRAM=38.6GiB | 72.3% done | ETA(epoch): 115s + Train loss: 0.863290 (1269.0s) ETA: 1940min + Val loss: 0.856141 [t_0.0-0.2=1.0725 t_0.2-0.4=0.9583 t_0.4-0.6=0.8421 t_0.6-0.8=0.6913 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 21.2/188.5 GiB (11.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 217/299 (72% done) --- + [Epoch 217][10/66] loss=0.926086 avg=0.871661 VRAM=38.7GiB | 72.4% done | ETA(epoch): 1075s + [Epoch 217][20/66] loss=0.773343 avg=0.869382 VRAM=38.6GiB | 72.4% done | ETA(epoch): 884s + [Epoch 217][30/66] loss=0.833142 avg=0.859815 VRAM=38.7GiB | 72.5% done | ETA(epoch): 692s + [Epoch 217][40/66] loss=0.787012 avg=0.858573 VRAM=38.6GiB | 72.5% done | ETA(epoch): 500s + [Epoch 217][50/66] loss=0.874145 avg=0.852216 VRAM=38.7GiB | 72.6% done | ETA(epoch): 307s + [Epoch 217][60/66] loss=0.760984 avg=0.851975 VRAM=38.6GiB | 72.6% done | ETA(epoch): 115s + Train loss: 0.854328 (1267.8s) ETA: 1917min + Val loss: 0.853327 [t_0.0-0.2=1.0623 t_0.2-0.4=1.0045 t_0.4-0.6=0.8459 t_0.6-0.8=0.7247 t_0.8-1.0=0.6355] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 218/299 (73% done) --- + [Epoch 218][10/66] loss=0.828719 avg=0.863748 VRAM=38.7GiB | 72.7% done | ETA(epoch): 1076s + [Epoch 218][20/66] loss=0.720785 avg=0.867195 VRAM=38.6GiB | 72.8% done | ETA(epoch): 888s + [Epoch 218][30/66] loss=0.747444 avg=0.855165 VRAM=38.7GiB | 72.8% done | ETA(epoch): 694s + [Epoch 218][40/66] loss=0.884584 avg=0.847479 VRAM=38.6GiB | 72.9% done | ETA(epoch): 501s + [Epoch 218][50/66] loss=0.946486 avg=0.852790 VRAM=38.7GiB | 72.9% done | ETA(epoch): 308s + [Epoch 218][60/66] loss=0.960174 avg=0.854254 VRAM=38.6GiB | 73.0% done | ETA(epoch): 115s + Train loss: 0.848607 (1269.5s) ETA: 1893min + Val loss: 0.845246 [t_0.0-0.2=1.0523 t_0.2-0.4=1.0028 t_0.4-0.6=0.8571 t_0.6-0.8=0.7188 t_0.8-1.0=0.6449] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 219/299 (73% done) --- + [Epoch 219][10/66] loss=0.812518 avg=0.862197 VRAM=38.7GiB | 73.1% done | ETA(epoch): 1087s + [Epoch 219][20/66] loss=0.955534 avg=0.866655 VRAM=38.6GiB | 73.1% done | ETA(epoch): 889s + [Epoch 219][30/66] loss=0.934166 avg=0.867482 VRAM=38.7GiB | 73.2% done | ETA(epoch): 694s + [Epoch 219][40/66] loss=0.768605 avg=0.866697 VRAM=38.6GiB | 73.2% done | ETA(epoch): 501s + [Epoch 219][50/66] loss=0.795234 avg=0.858025 VRAM=38.7GiB | 73.3% done | ETA(epoch): 308s + [Epoch 219][60/66] loss=0.905299 avg=0.855968 VRAM=38.6GiB | 73.3% done | ETA(epoch): 116s + Train loss: 0.858668 (1270.6s) ETA: 1869min + Val loss: 0.863473 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9942 t_0.4-0.6=0.8460 t_0.6-0.8=0.7019 t_0.8-1.0=0.6277] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 220/299 (73% done) --- + [Epoch 220][10/66] loss=0.772840 avg=0.799947 VRAM=38.7GiB | 73.4% done | ETA(epoch): 1087s + [Epoch 220][20/66] loss=0.710002 avg=0.825332 VRAM=38.6GiB | 73.4% done | ETA(epoch): 888s + [Epoch 220][30/66] loss=0.799463 avg=0.842530 VRAM=38.7GiB | 73.5% done | ETA(epoch): 693s + [Epoch 220][40/66] loss=0.961417 avg=0.846339 VRAM=38.6GiB | 73.5% done | ETA(epoch): 501s + [Epoch 220][50/66] loss=0.844929 avg=0.843276 VRAM=38.7GiB | 73.6% done | ETA(epoch): 308s + [Epoch 220][60/66] loss=0.968314 avg=0.851240 VRAM=38.6GiB | 73.6% done | ETA(epoch): 115s + Train loss: 0.854337 (1269.4s) ETA: 1846min + Val loss: 0.869582 [t_0.0-0.2=1.0635 t_0.2-0.4=1.0066 t_0.4-0.6=0.8627 t_0.6-0.8=0.7070 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 221/299 (74% done) --- + [Epoch 221][10/66] loss=0.849275 avg=0.829038 VRAM=38.7GiB | 73.7% done | ETA(epoch): 1078s + [Epoch 221][20/66] loss=0.876973 avg=0.858476 VRAM=38.6GiB | 73.8% done | ETA(epoch): 885s + [Epoch 221][30/66] loss=0.944091 avg=0.852264 VRAM=38.7GiB | 73.8% done | ETA(epoch): 692s + [Epoch 221][40/66] loss=0.833249 avg=0.851686 VRAM=38.6GiB | 73.9% done | ETA(epoch): 500s + [Epoch 221][50/66] loss=0.946993 avg=0.855521 VRAM=38.7GiB | 73.9% done | ETA(epoch): 307s + [Epoch 221][60/66] loss=0.950373 avg=0.859405 VRAM=38.6GiB | 74.0% done | ETA(epoch): 115s + Train loss: 0.855287 (1268.1s) ETA: 1822min + Val loss: 0.856875 [t_0.0-0.2=1.0605 t_0.2-0.4=1.0096 t_0.4-0.6=0.8495 t_0.6-0.8=0.7128 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 222/299 (74% done) --- + [Epoch 222][10/66] loss=0.751154 avg=0.859199 VRAM=38.7GiB | 74.1% done | ETA(epoch): 1075s + [Epoch 222][20/66] loss=0.971941 avg=0.832872 VRAM=38.6GiB | 74.1% done | ETA(epoch): 884s + [Epoch 222][30/66] loss=0.841418 avg=0.838430 VRAM=38.7GiB | 74.2% done | ETA(epoch): 692s + [Epoch 222][40/66] loss=0.841529 avg=0.845049 VRAM=38.6GiB | 74.2% done | ETA(epoch): 500s + [Epoch 222][50/66] loss=0.834353 avg=0.851338 VRAM=38.7GiB | 74.3% done | ETA(epoch): 308s + [Epoch 222][60/66] loss=0.790221 avg=0.854161 VRAM=38.6GiB | 74.3% done | ETA(epoch): 115s + Train loss: 0.850268 (1270.2s) ETA: 1799min + Val loss: 0.878712 [t_0.0-0.2=1.0625 t_0.2-0.4=0.9961 t_0.4-0.6=0.8605 t_0.6-0.8=0.7183 t_0.8-1.0=0.6298] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 223/299 (74% done) --- + [Epoch 223][10/66] loss=0.861197 avg=0.907084 VRAM=38.7GiB | 74.4% done | ETA(epoch): 1075s + [Epoch 223][20/66] loss=0.901859 avg=0.896382 VRAM=38.6GiB | 74.4% done | ETA(epoch): 882s + [Epoch 223][30/66] loss=0.819861 avg=0.882160 VRAM=38.7GiB | 74.5% done | ETA(epoch): 691s + [Epoch 223][40/66] loss=0.891597 avg=0.877477 VRAM=38.6GiB | 74.5% done | ETA(epoch): 502s + [Epoch 223][50/66] loss=0.820897 avg=0.878603 VRAM=38.7GiB | 74.6% done | ETA(epoch): 309s + [Epoch 223][60/66] loss=0.792737 avg=0.877743 VRAM=38.6GiB | 74.6% done | ETA(epoch): 116s + Train loss: 0.867464 (1272.2s) ETA: 1775min + Val loss: 0.859329 [t_0.0-0.2=1.0850 t_0.2-0.4=1.0056 t_0.4-0.6=0.8484 t_0.6-0.8=0.7152 t_0.8-1.0=0.6388] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 224/299 (75% done) --- + [Epoch 224][10/66] loss=0.875063 avg=0.881534 VRAM=38.7GiB | 74.7% done | ETA(epoch): 1078s + [Epoch 224][20/66] loss=1.012748 avg=0.857497 VRAM=38.6GiB | 74.8% done | ETA(epoch): 884s + [Epoch 224][30/66] loss=0.795699 avg=0.866987 VRAM=38.7GiB | 74.8% done | ETA(epoch): 692s + [Epoch 224][40/66] loss=0.842881 avg=0.865632 VRAM=38.6GiB | 74.9% done | ETA(epoch): 500s + [Epoch 224][50/66] loss=0.856408 avg=0.869074 VRAM=38.7GiB | 74.9% done | ETA(epoch): 307s + [Epoch 224][60/66] loss=0.758143 avg=0.868105 VRAM=38.6GiB | 75.0% done | ETA(epoch): 115s + Train loss: 0.866111 (1267.6s) ETA: 1752min + Val loss: 0.875421 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0008 t_0.4-0.6=0.8171 t_0.6-0.8=0.7129 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 225/299 (75% done) --- + [Epoch 225][10/66] loss=0.904849 avg=0.860473 VRAM=38.7GiB | 75.1% done | ETA(epoch): 1076s + [Epoch 225][20/66] loss=0.810630 avg=0.851964 VRAM=38.6GiB | 75.1% done | ETA(epoch): 883s + [Epoch 225][30/66] loss=0.844885 avg=0.852664 VRAM=38.7GiB | 75.2% done | ETA(epoch): 693s + [Epoch 225][40/66] loss=0.928674 avg=0.856296 VRAM=38.6GiB | 75.2% done | ETA(epoch): 500s + [Epoch 225][50/66] loss=0.927149 avg=0.856311 VRAM=38.7GiB | 75.3% done | ETA(epoch): 308s + [Epoch 225][60/66] loss=0.813589 avg=0.852638 VRAM=38.6GiB | 75.3% done | ETA(epoch): 115s + Train loss: 0.851644 (1268.9s) ETA: 1728min + Val loss: 0.840958 [t_0.0-0.2=1.0655 t_0.2-0.4=1.0022 t_0.4-0.6=0.8684 t_0.6-0.8=0.7015 t_0.8-1.0=0.6308] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 226/299 (75% done) --- + [Epoch 226][10/66] loss=0.915952 avg=0.878995 VRAM=38.7GiB | 75.4% done | ETA(epoch): 1074s + [Epoch 226][20/66] loss=0.914884 avg=0.869380 VRAM=38.6GiB | 75.4% done | ETA(epoch): 882s + [Epoch 226][30/66] loss=0.791517 avg=0.861874 VRAM=38.7GiB | 75.5% done | ETA(epoch): 691s + [Epoch 226][40/66] loss=0.846646 avg=0.862516 VRAM=38.6GiB | 75.5% done | ETA(epoch): 499s + [Epoch 226][50/66] loss=0.775628 avg=0.858529 VRAM=38.7GiB | 75.6% done | ETA(epoch): 307s + [Epoch 226][60/66] loss=0.802256 avg=0.861901 VRAM=38.6GiB | 75.6% done | ETA(epoch): 115s + Train loss: 0.862275 (1267.9s) ETA: 1704min + Val loss: 0.848807 [t_0.0-0.2=1.0822 t_0.2-0.4=1.0073 t_0.4-0.6=0.8050 t_0.6-0.8=0.7341 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 227/299 (76% done) --- + [Epoch 227][10/66] loss=0.862175 avg=0.899143 VRAM=38.7GiB | 75.7% done | ETA(epoch): 1098s + [Epoch 227][20/66] loss=0.963315 avg=0.867513 VRAM=38.6GiB | 75.8% done | ETA(epoch): 893s + [Epoch 227][30/66] loss=0.810034 avg=0.867838 VRAM=38.7GiB | 75.8% done | ETA(epoch): 697s + [Epoch 227][40/66] loss=0.887836 avg=0.866887 VRAM=38.6GiB | 75.9% done | ETA(epoch): 502s + [Epoch 227][50/66] loss=0.731836 avg=0.862240 VRAM=38.7GiB | 75.9% done | ETA(epoch): 309s + [Epoch 227][60/66] loss=0.840907 avg=0.856186 VRAM=38.6GiB | 76.0% done | ETA(epoch): 116s + Train loss: 0.853320 (1271.8s) ETA: 1681min + Val loss: 0.838132 [t_0.0-0.2=1.0872 t_0.2-0.4=0.9881 t_0.4-0.6=0.8676 t_0.6-0.8=0.6882 t_0.8-1.0=0.6400] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 228/299 (76% done) --- + [Epoch 228][10/66] loss=0.846885 avg=0.872752 VRAM=38.7GiB | 76.1% done | ETA(epoch): 1074s + [Epoch 228][20/66] loss=0.865942 avg=0.869388 VRAM=38.6GiB | 76.1% done | ETA(epoch): 883s + [Epoch 228][30/66] loss=0.991987 avg=0.872107 VRAM=38.7GiB | 76.2% done | ETA(epoch): 691s + [Epoch 228][40/66] loss=0.883305 avg=0.866276 VRAM=38.6GiB | 76.2% done | ETA(epoch): 499s + [Epoch 228][50/66] loss=0.812380 avg=0.858902 VRAM=38.7GiB | 76.3% done | ETA(epoch): 307s + [Epoch 228][60/66] loss=0.858859 avg=0.855681 VRAM=38.6GiB | 76.3% done | ETA(epoch): 115s + Train loss: 0.857360 (1271.9s) ETA: 1657min + Val loss: 0.844468 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9900 t_0.4-0.6=0.8643 t_0.6-0.8=0.7068 t_0.8-1.0=0.6405] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 229/299 (76% done) --- + [Epoch 229][10/66] loss=0.848713 avg=0.852210 VRAM=38.7GiB | 76.4% done | ETA(epoch): 1074s + [Epoch 229][20/66] loss=0.955469 avg=0.859190 VRAM=38.6GiB | 76.4% done | ETA(epoch): 883s + [Epoch 229][30/66] loss=0.953135 avg=0.861338 VRAM=38.7GiB | 76.5% done | ETA(epoch): 692s + [Epoch 229][40/66] loss=0.814107 avg=0.863321 VRAM=38.6GiB | 76.5% done | ETA(epoch): 500s + [Epoch 229][50/66] loss=0.888357 avg=0.860039 VRAM=38.7GiB | 76.6% done | ETA(epoch): 307s + [Epoch 229][60/66] loss=0.836408 avg=0.852344 VRAM=38.6GiB | 76.6% done | ETA(epoch): 116s + Train loss: 0.849423 (1270.7s) ETA: 1634min + Val loss: 0.860125 [t_0.0-0.2=1.0851 t_0.2-0.4=1.0052 t_0.4-0.6=0.8304 t_0.6-0.8=0.7037 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 230/299 (77% done) --- + [Epoch 230][10/66] loss=0.932716 avg=0.877739 VRAM=38.7GiB | 76.7% done | ETA(epoch): 1077s + [Epoch 230][20/66] loss=0.872596 avg=0.873151 VRAM=38.6GiB | 76.8% done | ETA(epoch): 885s + [Epoch 230][30/66] loss=0.971305 avg=0.860515 VRAM=38.7GiB | 76.8% done | ETA(epoch): 692s + [Epoch 230][40/66] loss=0.954962 avg=0.866083 VRAM=38.6GiB | 76.9% done | ETA(epoch): 500s + [Epoch 230][50/66] loss=0.779808 avg=0.863074 VRAM=38.7GiB | 76.9% done | ETA(epoch): 309s + [Epoch 230][60/66] loss=0.875721 avg=0.861574 VRAM=38.6GiB | 77.0% done | ETA(epoch): 116s + Train loss: 0.860834 (1271.5s) ETA: 1610min + Val loss: 0.854880 [t_0.0-0.2=1.0924 t_0.2-0.4=0.9772 t_0.4-0.6=0.8525 t_0.6-0.8=0.6995 t_0.8-1.0=0.6454] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 231/299 (77% done) --- + [Epoch 231][10/66] loss=0.709717 avg=0.877166 VRAM=38.7GiB | 77.1% done | ETA(epoch): 1075s + [Epoch 231][20/66] loss=0.822698 avg=0.871149 VRAM=38.6GiB | 77.1% done | ETA(epoch): 884s + [Epoch 231][30/66] loss=0.883106 avg=0.865276 VRAM=38.7GiB | 77.2% done | ETA(epoch): 692s + [Epoch 231][40/66] loss=0.877157 avg=0.854933 VRAM=38.6GiB | 77.2% done | ETA(epoch): 501s + [Epoch 231][50/66] loss=0.787557 avg=0.859694 VRAM=38.7GiB | 77.3% done | ETA(epoch): 308s + [Epoch 231][60/66] loss=0.910379 avg=0.858099 VRAM=38.6GiB | 77.3% done | ETA(epoch): 115s + Train loss: 0.853756 (1269.6s) ETA: 1587min + Val loss: 0.848906 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0023 t_0.4-0.6=0.8439 t_0.6-0.8=0.7040 t_0.8-1.0=0.6302] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 232/299 (77% done) --- + [Epoch 232][10/66] loss=0.981920 avg=0.872498 VRAM=38.7GiB | 77.4% done | ETA(epoch): 1077s + [Epoch 232][20/66] loss=0.819133 avg=0.870437 VRAM=38.6GiB | 77.4% done | ETA(epoch): 884s + [Epoch 232][30/66] loss=0.693995 avg=0.857444 VRAM=38.7GiB | 77.5% done | ETA(epoch): 692s + [Epoch 232][40/66] loss=0.811225 avg=0.860048 VRAM=38.6GiB | 77.5% done | ETA(epoch): 500s + [Epoch 232][50/66] loss=0.877437 avg=0.857776 VRAM=38.7GiB | 77.6% done | ETA(epoch): 307s + [Epoch 232][60/66] loss=0.842139 avg=0.859325 VRAM=38.6GiB | 77.6% done | ETA(epoch): 115s + Train loss: 0.857073 (1268.1s) ETA: 1563min + Val loss: 0.876365 [t_0.0-0.2=1.0581 t_0.2-0.4=1.0116 t_0.4-0.6=0.8628 t_0.6-0.8=0.6989 t_0.8-1.0=0.6565] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 233/299 (78% done) --- + [Epoch 233][10/66] loss=0.871242 avg=0.887003 VRAM=38.7GiB | 77.7% done | ETA(epoch): 1075s + [Epoch 233][20/66] loss=0.892603 avg=0.869069 VRAM=38.6GiB | 77.8% done | ETA(epoch): 883s + [Epoch 233][30/66] loss=0.853064 avg=0.864630 VRAM=38.7GiB | 77.8% done | ETA(epoch): 692s + [Epoch 233][40/66] loss=0.847269 avg=0.861453 VRAM=38.6GiB | 77.9% done | ETA(epoch): 500s + [Epoch 233][50/66] loss=0.781307 avg=0.851486 VRAM=38.7GiB | 77.9% done | ETA(epoch): 307s + [Epoch 233][60/66] loss=0.880711 avg=0.852586 VRAM=38.6GiB | 78.0% done | ETA(epoch): 115s + Train loss: 0.847541 (1267.8s) ETA: 1540min + Val loss: 0.877529 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9640 t_0.4-0.6=0.8551 t_0.6-0.8=0.6874 t_0.8-1.0=0.6347] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 234/299 (78% done) --- + [Epoch 234][10/66] loss=0.752102 avg=0.912347 VRAM=38.7GiB | 78.1% done | ETA(epoch): 1074s + [Epoch 234][20/66] loss=0.814392 avg=0.886429 VRAM=38.6GiB | 78.1% done | ETA(epoch): 889s + [Epoch 234][30/66] loss=0.943109 avg=0.882204 VRAM=38.7GiB | 78.2% done | ETA(epoch): 694s + [Epoch 234][40/66] loss=0.868618 avg=0.872180 VRAM=38.6GiB | 78.2% done | ETA(epoch): 501s + [Epoch 234][50/66] loss=0.827305 avg=0.858890 VRAM=38.7GiB | 78.3% done | ETA(epoch): 308s + [Epoch 234][60/66] loss=0.753465 avg=0.853582 VRAM=38.6GiB | 78.3% done | ETA(epoch): 115s + Train loss: 0.853337 (1269.8s) ETA: 1516min + Val loss: 0.848464 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9886 t_0.4-0.6=0.8659 t_0.6-0.8=0.7013 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 21.2/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 235/299 (78% done) --- + [Epoch 235][10/66] loss=0.910781 avg=0.882386 VRAM=38.7GiB | 78.4% done | ETA(epoch): 1084s + [Epoch 235][20/66] loss=0.966199 avg=0.852650 VRAM=38.6GiB | 78.4% done | ETA(epoch): 887s + [Epoch 235][30/66] loss=0.912078 avg=0.848315 VRAM=38.7GiB | 78.5% done | ETA(epoch): 693s + [Epoch 235][40/66] loss=0.712090 avg=0.839851 VRAM=38.6GiB | 78.5% done | ETA(epoch): 500s + [Epoch 235][50/66] loss=0.873284 avg=0.839949 VRAM=38.7GiB | 78.6% done | ETA(epoch): 308s + [Epoch 235][60/66] loss=0.903980 avg=0.843254 VRAM=38.6GiB | 78.6% done | ETA(epoch): 115s + Train loss: 0.843725 (1270.9s) ETA: 1493min + Val loss: 0.869037 [t_0.0-0.2=1.0688 t_0.2-0.4=1.0055 t_0.4-0.6=0.8392 t_0.6-0.8=0.7173 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 236/299 (79% done) --- + [Epoch 236][10/66] loss=0.872937 avg=0.875758 VRAM=38.7GiB | 78.7% done | ETA(epoch): 1074s + [Epoch 236][20/66] loss=0.869493 avg=0.871318 VRAM=38.6GiB | 78.8% done | ETA(epoch): 883s + [Epoch 236][30/66] loss=0.799229 avg=0.876379 VRAM=38.7GiB | 78.8% done | ETA(epoch): 692s + [Epoch 236][40/66] loss=0.771659 avg=0.856064 VRAM=38.6GiB | 78.9% done | ETA(epoch): 499s + [Epoch 236][50/66] loss=0.907237 avg=0.856337 VRAM=38.7GiB | 78.9% done | ETA(epoch): 307s + [Epoch 236][60/66] loss=0.852203 avg=0.856495 VRAM=38.6GiB | 79.0% done | ETA(epoch): 116s + Train loss: 0.856412 (1270.3s) ETA: 1469min + Val loss: 0.863682 [t_0.0-0.2=1.0701 t_0.2-0.4=0.9870 t_0.4-0.6=0.8966 t_0.6-0.8=0.6899 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 237/299 (79% done) --- + [Epoch 237][10/66] loss=0.812350 avg=0.826718 VRAM=38.7GiB | 79.1% done | ETA(epoch): 1079s + [Epoch 237][20/66] loss=0.870323 avg=0.836763 VRAM=38.6GiB | 79.1% done | ETA(epoch): 885s + [Epoch 237][30/66] loss=0.985646 avg=0.844654 VRAM=38.7GiB | 79.2% done | ETA(epoch): 692s + [Epoch 237][40/66] loss=0.711072 avg=0.844246 VRAM=38.6GiB | 79.2% done | ETA(epoch): 500s + [Epoch 237][50/66] loss=0.776398 avg=0.840180 VRAM=38.7GiB | 79.3% done | ETA(epoch): 308s + [Epoch 237][60/66] loss=0.856292 avg=0.840884 VRAM=38.6GiB | 79.3% done | ETA(epoch): 115s + Train loss: 0.842491 (1269.0s) ETA: 1446min + Val loss: 0.851034 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9809 t_0.4-0.6=0.8680 t_0.6-0.8=0.7319 t_0.8-1.0=0.6498] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 238/299 (79% done) --- + [Epoch 238][10/66] loss=0.827730 avg=0.867073 VRAM=38.7GiB | 79.4% done | ETA(epoch): 1075s + [Epoch 238][20/66] loss=0.974059 avg=0.832639 VRAM=38.6GiB | 79.4% done | ETA(epoch): 883s + [Epoch 238][30/66] loss=0.746249 avg=0.834185 VRAM=38.7GiB | 79.5% done | ETA(epoch): 691s + [Epoch 238][40/66] loss=0.704199 avg=0.831796 VRAM=38.6GiB | 79.5% done | ETA(epoch): 499s + [Epoch 238][50/66] loss=0.898779 avg=0.839108 VRAM=38.7GiB | 79.6% done | ETA(epoch): 308s + [Epoch 238][60/66] loss=0.971033 avg=0.841139 VRAM=38.6GiB | 79.6% done | ETA(epoch): 116s + Train loss: 0.840394 (1270.3s) ETA: 1422min + Val loss: 0.865844 [t_0.0-0.2=1.0550 t_0.2-0.4=1.0107 t_0.4-0.6=0.8473 t_0.6-0.8=0.7171 t_0.8-1.0=0.6343] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 239/299 (80% done) --- + [Epoch 239][10/66] loss=0.895410 avg=0.845141 VRAM=38.7GiB | 79.7% done | ETA(epoch): 1076s + [Epoch 239][20/66] loss=0.751563 avg=0.839433 VRAM=38.6GiB | 79.8% done | ETA(epoch): 884s + [Epoch 239][30/66] loss=0.815522 avg=0.841615 VRAM=38.7GiB | 79.8% done | ETA(epoch): 691s + [Epoch 239][40/66] loss=0.887749 avg=0.852075 VRAM=38.6GiB | 79.9% done | ETA(epoch): 501s + [Epoch 239][50/66] loss=0.861113 avg=0.854157 VRAM=38.7GiB | 79.9% done | ETA(epoch): 308s + [Epoch 239][60/66] loss=0.891223 avg=0.853133 VRAM=38.6GiB | 80.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 15839 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 15839 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 15839 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 15839 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 15839 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 15839 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 15839 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 15839 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 15839 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 15839 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.854206 (2306.2s) ETA: 1403min + Val loss: 0.876596 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9969 t_0.4-0.6=0.8246 t_0.6-0.8=0.7089 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 240/299 (80% done) --- + [Epoch 240][10/66] loss=0.787140 avg=0.863296 VRAM=38.7GiB | 80.1% done | ETA(epoch): 1076s + [Epoch 240][20/66] loss=0.774624 avg=0.837799 VRAM=38.6GiB | 80.1% done | ETA(epoch): 884s + [Epoch 240][30/66] loss=0.703705 avg=0.838369 VRAM=38.7GiB | 80.2% done | ETA(epoch): 692s + [Epoch 240][40/66] loss=0.895897 avg=0.845769 VRAM=38.6GiB | 80.2% done | ETA(epoch): 502s + [Epoch 240][50/66] loss=0.842433 avg=0.848904 VRAM=38.7GiB | 80.3% done | ETA(epoch): 308s + [Epoch 240][60/66] loss=0.809377 avg=0.845360 VRAM=38.6GiB | 80.3% done | ETA(epoch): 116s + Train loss: 0.839855 (1271.7s) ETA: 1380min + Val loss: 0.878801 [t_0.0-0.2=1.0652 t_0.2-0.4=0.9927 t_0.4-0.6=0.8466 t_0.6-0.8=0.6924 t_0.8-1.0=0.6624] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 241/299 (80% done) --- + [Epoch 241][10/66] loss=0.912179 avg=0.857497 VRAM=38.7GiB | 80.4% done | ETA(epoch): 1075s + [Epoch 241][20/66] loss=0.767609 avg=0.870679 VRAM=38.6GiB | 80.4% done | ETA(epoch): 884s + [Epoch 241][30/66] loss=0.869425 avg=0.852299 VRAM=38.7GiB | 80.5% done | ETA(epoch): 694s + [Epoch 241][40/66] loss=0.741338 avg=0.843988 VRAM=38.6GiB | 80.5% done | ETA(epoch): 501s + [Epoch 241][50/66] loss=0.785167 avg=0.840787 VRAM=38.7GiB | 80.6% done | ETA(epoch): 308s + [Epoch 241][60/66] loss=0.846666 avg=0.842914 VRAM=38.6GiB | 80.6% done | ETA(epoch): 115s + Train loss: 0.843116 (1269.9s) ETA: 1356min + Val loss: 0.834031 [t_0.0-0.2=1.0598 t_0.2-0.4=0.9754 t_0.4-0.6=0.8433 t_0.6-0.8=0.7098 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 22.0/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 242/299 (81% done) --- + [Epoch 242][10/66] loss=0.870358 avg=0.846161 VRAM=38.7GiB | 80.7% done | ETA(epoch): 1075s + [Epoch 242][20/66] loss=0.945001 avg=0.877912 VRAM=38.6GiB | 80.8% done | ETA(epoch): 883s + [Epoch 242][30/66] loss=0.813352 avg=0.861710 VRAM=38.7GiB | 80.8% done | ETA(epoch): 692s + [Epoch 242][40/66] loss=0.998891 avg=0.864450 VRAM=38.6GiB | 80.9% done | ETA(epoch): 499s + [Epoch 242][50/66] loss=0.811622 avg=0.864600 VRAM=38.7GiB | 80.9% done | ETA(epoch): 307s + [Epoch 242][60/66] loss=0.838374 avg=0.863397 VRAM=38.6GiB | 81.0% done | ETA(epoch): 115s + Train loss: 0.863520 (1267.7s) ETA: 1333min + Val loss: 0.846071 [t_0.0-0.2=1.0724 t_0.2-0.4=1.0207 t_0.4-0.6=0.8473 t_0.6-0.8=0.7152 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 243/299 (81% done) --- + [Epoch 243][10/66] loss=0.920121 avg=0.849375 VRAM=38.7GiB | 81.1% done | ETA(epoch): 1076s + [Epoch 243][20/66] loss=0.905439 avg=0.834093 VRAM=38.6GiB | 81.1% done | ETA(epoch): 884s + [Epoch 243][30/66] loss=0.860592 avg=0.852482 VRAM=38.7GiB | 81.2% done | ETA(epoch): 692s + [Epoch 243][40/66] loss=0.708679 avg=0.855100 VRAM=38.6GiB | 81.2% done | ETA(epoch): 500s + [Epoch 243][50/66] loss=0.696346 avg=0.848628 VRAM=38.7GiB | 81.3% done | ETA(epoch): 308s + [Epoch 243][60/66] loss=0.720488 avg=0.842127 VRAM=38.6GiB | 81.3% done | ETA(epoch): 115s + Train loss: 0.846170 (1268.8s) ETA: 1309min + Val loss: 0.850091 [t_0.0-0.2=1.0850 t_0.2-0.4=0.9981 t_0.4-0.6=0.8732 t_0.6-0.8=0.7137 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 22.0/188.5 GiB (11.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 244/299 (81% done) --- + [Epoch 244][10/66] loss=0.729012 avg=0.852176 VRAM=38.7GiB | 81.4% done | ETA(epoch): 1076s + [Epoch 244][20/66] loss=0.843952 avg=0.878362 VRAM=38.6GiB | 81.4% done | ETA(epoch): 884s + [Epoch 244][30/66] loss=0.853844 avg=0.882030 VRAM=38.7GiB | 81.5% done | ETA(epoch): 692s + [Epoch 244][40/66] loss=0.838826 avg=0.873096 VRAM=38.6GiB | 81.5% done | ETA(epoch): 500s + [Epoch 244][50/66] loss=0.858385 avg=0.861785 VRAM=38.7GiB | 81.6% done | ETA(epoch): 308s + [Epoch 244][60/66] loss=0.868736 avg=0.863985 VRAM=38.6GiB | 81.6% done | ETA(epoch): 115s + Train loss: 0.863866 (1270.7s) ETA: 1286min + Val loss: 0.852134 [t_0.0-0.2=1.0810 t_0.2-0.4=1.0148 t_0.4-0.6=0.8295 t_0.6-0.8=0.7031 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 245/299 (82% done) --- + [Epoch 245][10/66] loss=0.937625 avg=0.846600 VRAM=38.7GiB | 81.7% done | ETA(epoch): 1074s + [Epoch 245][20/66] loss=0.778574 avg=0.866680 VRAM=38.6GiB | 81.8% done | ETA(epoch): 883s + [Epoch 245][30/66] loss=0.895449 avg=0.868537 VRAM=38.7GiB | 81.8% done | ETA(epoch): 691s + [Epoch 245][40/66] loss=0.913369 avg=0.858557 VRAM=38.6GiB | 81.9% done | ETA(epoch): 499s + [Epoch 245][50/66] loss=0.825185 avg=0.857480 VRAM=38.7GiB | 81.9% done | ETA(epoch): 307s + [Epoch 245][60/66] loss=0.889707 avg=0.853811 VRAM=38.6GiB | 82.0% done | ETA(epoch): 115s + Train loss: 0.855615 (1268.5s) ETA: 1262min + Val loss: 0.849604 [t_0.0-0.2=1.0487 t_0.2-0.4=1.0167 t_0.4-0.6=0.8699 t_0.6-0.8=0.6969 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0245 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 21.9/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 246/299 (82% done) --- + [Epoch 246][10/66] loss=0.862990 avg=0.885863 VRAM=38.7GiB | 82.1% done | ETA(epoch): 1075s + [Epoch 246][20/66] loss=0.824605 avg=0.886797 VRAM=38.6GiB | 82.1% done | ETA(epoch): 883s + [Epoch 246][30/66] loss=0.969709 avg=0.886833 VRAM=38.7GiB | 82.2% done | ETA(epoch): 692s + [Epoch 246][40/66] loss=0.942387 avg=0.872617 VRAM=38.6GiB | 82.2% done | ETA(epoch): 500s + [Epoch 246][50/66] loss=0.853779 avg=0.881030 VRAM=38.7GiB | 82.3% done | ETA(epoch): 308s + [Epoch 246][60/66] loss=0.862735 avg=0.878630 VRAM=38.6GiB | 82.3% done | ETA(epoch): 116s + Train loss: 0.871550 (1270.8s) ETA: 1239min + Val loss: 0.860198 [t_0.0-0.2=1.0532 t_0.2-0.4=1.0071 t_0.4-0.6=0.8610 t_0.6-0.8=0.7063 t_0.8-1.0=0.6442] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 22.0/188.5 GiB (11.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 247/299 (82% done) --- + [Epoch 247][10/66] loss=0.894396 avg=0.864008 VRAM=38.7GiB | 82.4% done | ETA(epoch): 1076s + [Epoch 247][20/66] loss=0.888493 avg=0.880235 VRAM=38.6GiB | 82.4% done | ETA(epoch): 884s + [Epoch 247][30/66] loss=0.782939 avg=0.876776 VRAM=38.7GiB | 82.5% done | ETA(epoch): 692s + [Epoch 247][40/66] loss=0.822438 avg=0.868351 VRAM=38.6GiB | 82.5% done | ETA(epoch): 500s + [Epoch 247][50/66] loss=0.967803 avg=0.868444 VRAM=38.7GiB | 82.6% done | ETA(epoch): 307s + [Epoch 247][60/66] loss=0.807544 avg=0.859738 VRAM=38.6GiB | 82.6% done | ETA(epoch): 115s + Train loss: 0.862784 (1268.6s) ETA: 1215min + Val loss: 0.833421 [t_0.0-0.2=1.0803 t_0.2-0.4=0.9697 t_0.4-0.6=0.8614 t_0.6-0.8=0.7071 t_0.8-1.0=0.6324] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 22.4/188.5 GiB (11.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 248/299 (83% done) --- + [Epoch 248][10/66] loss=0.874666 avg=0.802905 VRAM=38.7GiB | 82.7% done | ETA(epoch): 1075s + [Epoch 248][20/66] loss=0.765164 avg=0.815018 VRAM=38.6GiB | 82.8% done | ETA(epoch): 884s + [Epoch 248][30/66] loss=0.877772 avg=0.816061 VRAM=38.7GiB | 82.8% done | ETA(epoch): 692s + [Epoch 248][40/66] loss=0.890725 avg=0.824597 VRAM=38.6GiB | 82.9% done | ETA(epoch): 500s + [Epoch 248][50/66] loss=0.807145 avg=0.827525 VRAM=38.7GiB | 82.9% done | ETA(epoch): 308s + [Epoch 248][60/66] loss=0.873834 avg=0.832708 VRAM=38.6GiB | 83.0% done | ETA(epoch): 115s + Train loss: 0.838171 (1268.9s) ETA: 1192min + Val loss: 0.871448 [t_0.0-0.2=1.0638 t_0.2-0.4=0.9816 t_0.4-0.6=0.8443 t_0.6-0.8=0.7140 t_0.8-1.0=0.6440] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0248 + Deleted old checkpoint: checkpoint_epoch_0245 +[MEM @ epoch 248 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 249/299 (83% done) --- + [Epoch 249][10/66] loss=0.796068 avg=0.858691 VRAM=38.7GiB | 83.1% done | ETA(epoch): 1075s + [Epoch 249][20/66] loss=0.782815 avg=0.846773 VRAM=38.6GiB | 83.1% done | ETA(epoch): 883s + [Epoch 249][30/66] loss=0.847260 avg=0.848135 VRAM=38.7GiB | 83.2% done | ETA(epoch): 692s + [Epoch 249][40/66] loss=0.760924 avg=0.848605 VRAM=38.6GiB | 83.2% done | ETA(epoch): 500s + [Epoch 249][50/66] loss=0.904863 avg=0.840797 VRAM=38.7GiB | 83.3% done | ETA(epoch): 307s + [Epoch 249][60/66] loss=0.925921 avg=0.845903 VRAM=38.6GiB | 83.3% done | ETA(epoch): 115s + Train loss: 0.848154 (1268.0s) ETA: 1168min + Val loss: 0.841187 [t_0.0-0.2=1.0428 t_0.2-0.4=0.9799 t_0.4-0.6=0.8425 t_0.6-0.8=0.7372 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 23.6/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 250/299 (83% done) --- + [Epoch 250][10/66] loss=0.894084 avg=0.870129 VRAM=38.7GiB | 83.4% done | ETA(epoch): 1076s + [Epoch 250][20/66] loss=0.818025 avg=0.871779 VRAM=38.6GiB | 83.4% done | ETA(epoch): 884s + [Epoch 250][30/66] loss=0.886565 avg=0.872386 VRAM=38.7GiB | 83.5% done | ETA(epoch): 692s + [Epoch 250][40/66] loss=0.885947 avg=0.862387 VRAM=38.6GiB | 83.5% done | ETA(epoch): 500s + [Epoch 250][50/66] loss=0.765325 avg=0.858533 VRAM=38.7GiB | 83.6% done | ETA(epoch): 307s + [Epoch 250][60/66] loss=0.825293 avg=0.860608 VRAM=38.6GiB | 83.6% done | ETA(epoch): 115s + Train loss: 0.864750 (1268.1s) ETA: 1145min + Val loss: 0.873578 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0374 t_0.4-0.6=0.8140 t_0.6-0.8=0.7028 t_0.8-1.0=0.6560] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 251/299 (84% done) --- + [Epoch 251][10/66] loss=0.791023 avg=0.848134 VRAM=38.7GiB | 83.7% done | ETA(epoch): 1088s + [Epoch 251][20/66] loss=0.904123 avg=0.854735 VRAM=38.6GiB | 83.8% done | ETA(epoch): 888s + [Epoch 251][30/66] loss=1.000804 avg=0.851481 VRAM=38.7GiB | 83.8% done | ETA(epoch): 694s + [Epoch 251][40/66] loss=0.896419 avg=0.842626 VRAM=38.6GiB | 83.9% done | ETA(epoch): 501s + [Epoch 251][50/66] loss=0.874453 avg=0.849305 VRAM=38.7GiB | 83.9% done | ETA(epoch): 308s + [Epoch 251][60/66] loss=0.870213 avg=0.850910 VRAM=38.6GiB | 84.0% done | ETA(epoch): 116s + Train loss: 0.851227 (1270.5s) ETA: 1121min + Val loss: 0.834536 [t_0.0-0.2=1.0689 t_0.2-0.4=1.0130 t_0.4-0.6=0.8166 t_0.6-0.8=0.7057 t_0.8-1.0=0.6288] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 252/299 (84% done) --- + [Epoch 252][10/66] loss=0.900010 avg=0.843484 VRAM=38.7GiB | 84.1% done | ETA(epoch): 1076s + [Epoch 252][20/66] loss=0.962385 avg=0.845389 VRAM=38.6GiB | 84.1% done | ETA(epoch): 883s + [Epoch 252][30/66] loss=0.747069 avg=0.844062 VRAM=38.7GiB | 84.2% done | ETA(epoch): 691s + [Epoch 252][40/66] loss=0.773598 avg=0.842093 VRAM=38.6GiB | 84.2% done | ETA(epoch): 499s + [Epoch 252][50/66] loss=0.690155 avg=0.849126 VRAM=38.7GiB | 84.3% done | ETA(epoch): 307s + [Epoch 252][60/66] loss=0.964346 avg=0.856985 VRAM=38.6GiB | 84.3% done | ETA(epoch): 115s + Train loss: 0.856366 (1267.5s) ETA: 1098min + Val loss: 0.840749 [t_0.0-0.2=1.0656 t_0.2-0.4=1.0032 t_0.4-0.6=0.8322 t_0.6-0.8=0.7204 t_0.8-1.0=0.6333] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 23.4/188.5 GiB (12.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 253/299 (84% done) --- + [Epoch 253][10/66] loss=0.684275 avg=0.837892 VRAM=38.7GiB | 84.4% done | ETA(epoch): 1076s + [Epoch 253][20/66] loss=0.858235 avg=0.837075 VRAM=38.6GiB | 84.4% done | ETA(epoch): 883s + [Epoch 253][30/66] loss=0.903411 avg=0.858805 VRAM=38.7GiB | 84.5% done | ETA(epoch): 692s + [Epoch 253][40/66] loss=0.804553 avg=0.851308 VRAM=38.6GiB | 84.5% done | ETA(epoch): 500s + [Epoch 253][50/66] loss=0.903456 avg=0.854131 VRAM=38.7GiB | 84.6% done | ETA(epoch): 307s + [Epoch 253][60/66] loss=0.828712 avg=0.849585 VRAM=38.6GiB | 84.6% done | ETA(epoch): 115s + Train loss: 0.851042 (1268.1s) ETA: 1074min + Val loss: 0.848379 [t_0.0-0.2=1.0712 t_0.2-0.4=0.9921 t_0.4-0.6=0.8437 t_0.6-0.8=0.6995 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 254/299 (85% done) --- + [Epoch 254][10/66] loss=0.809953 avg=0.841180 VRAM=38.7GiB | 84.7% done | ETA(epoch): 1077s + [Epoch 254][20/66] loss=0.760078 avg=0.847949 VRAM=38.6GiB | 84.8% done | ETA(epoch): 884s + [Epoch 254][30/66] loss=0.817141 avg=0.848210 VRAM=38.7GiB | 84.8% done | ETA(epoch): 692s + [Epoch 254][40/66] loss=0.829085 avg=0.857483 VRAM=38.6GiB | 84.9% done | ETA(epoch): 500s + [Epoch 254][50/66] loss=0.957682 avg=0.859434 VRAM=38.7GiB | 84.9% done | ETA(epoch): 308s + [Epoch 254][60/66] loss=0.906189 avg=0.855983 VRAM=38.6GiB | 85.0% done | ETA(epoch): 115s + Train loss: 0.859093 (1269.0s) ETA: 1051min + Val loss: 0.880886 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0022 t_0.4-0.6=0.8568 t_0.6-0.8=0.7080 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 255/299 (85% done) --- + [Epoch 255][10/66] loss=0.843677 avg=0.853641 VRAM=38.7GiB | 85.1% done | ETA(epoch): 1074s + [Epoch 255][20/66] loss=0.774453 avg=0.840581 VRAM=38.6GiB | 85.1% done | ETA(epoch): 883s + [Epoch 255][30/66] loss=0.883752 avg=0.855174 VRAM=38.7GiB | 85.2% done | ETA(epoch): 691s + [Epoch 255][40/66] loss=0.928497 avg=0.858075 VRAM=38.6GiB | 85.2% done | ETA(epoch): 501s + [Epoch 255][50/66] loss=0.856970 avg=0.856864 VRAM=38.7GiB | 85.3% done | ETA(epoch): 308s + [Epoch 255][60/66] loss=0.909565 avg=0.855406 VRAM=38.6GiB | 85.3% done | ETA(epoch): 115s + Train loss: 0.856049 (1269.6s) ETA: 1027min + Val loss: 0.872086 [t_0.0-0.2=1.0628 t_0.2-0.4=0.9932 t_0.4-0.6=0.8627 t_0.6-0.8=0.7118 t_0.8-1.0=0.6341] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 23.2/188.5 GiB (12.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 256/299 (85% done) --- + [Epoch 256][10/66] loss=0.981425 avg=0.846604 VRAM=38.7GiB | 85.4% done | ETA(epoch): 1075s + [Epoch 256][20/66] loss=0.957981 avg=0.844380 VRAM=38.6GiB | 85.4% done | ETA(epoch): 883s + [Epoch 256][30/66] loss=0.954757 avg=0.844366 VRAM=38.7GiB | 85.5% done | ETA(epoch): 691s + [Epoch 256][40/66] loss=0.812134 avg=0.847590 VRAM=38.6GiB | 85.5% done | ETA(epoch): 499s + [Epoch 256][50/66] loss=0.942746 avg=0.848590 VRAM=38.7GiB | 85.6% done | ETA(epoch): 307s + [Epoch 256][60/66] loss=0.873548 avg=0.851288 VRAM=38.6GiB | 85.6% done | ETA(epoch): 115s + Train loss: 0.851761 (1268.2s) ETA: 1004min + Val loss: 0.859093 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9861 t_0.4-0.6=0.8485 t_0.6-0.8=0.7132 t_0.8-1.0=0.6479] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 23.4/188.5 GiB (12.4%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 257/299 (86% done) --- + [Epoch 257][10/66] loss=0.817333 avg=0.853536 VRAM=38.7GiB | 85.7% done | ETA(epoch): 1074s + [Epoch 257][20/66] loss=0.945451 avg=0.855219 VRAM=38.6GiB | 85.8% done | ETA(epoch): 883s + [Epoch 257][30/66] loss=0.820212 avg=0.845665 VRAM=38.7GiB | 85.8% done | ETA(epoch): 691s + [Epoch 257][40/66] loss=0.815575 avg=0.841267 VRAM=38.6GiB | 85.9% done | ETA(epoch): 499s + [Epoch 257][50/66] loss=0.927717 avg=0.846238 VRAM=38.7GiB | 85.9% done | ETA(epoch): 307s + [Epoch 257][60/66] loss=0.884299 avg=0.855853 VRAM=38.6GiB | 86.0% done | ETA(epoch): 115s + Train loss: 0.852596 (1267.0s) ETA: 980min + Val loss: 0.840893 [t_0.0-0.2=1.0590 t_0.2-0.4=1.0291 t_0.4-0.6=0.8365 t_0.6-0.8=0.7074 t_0.8-1.0=0.6237] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 22.9/188.5 GiB (12.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 258/299 (86% done) --- + [Epoch 258][10/66] loss=0.952319 avg=0.907028 VRAM=38.7GiB | 86.1% done | ETA(epoch): 1076s + [Epoch 258][20/66] loss=0.724133 avg=0.872093 VRAM=38.6GiB | 86.1% done | ETA(epoch): 884s + [Epoch 258][30/66] loss=0.763557 avg=0.856502 VRAM=38.7GiB | 86.2% done | ETA(epoch): 691s + [Epoch 258][40/66] loss=0.857859 avg=0.856379 VRAM=38.6GiB | 86.2% done | ETA(epoch): 499s + [Epoch 258][50/66] loss=0.939018 avg=0.860832 VRAM=38.7GiB | 86.3% done | ETA(epoch): 307s + [Epoch 258][60/66] loss=0.865154 avg=0.859913 VRAM=38.6GiB | 86.3% done | ETA(epoch): 115s + Train loss: 0.855585 (1268.2s) ETA: 957min + Val loss: 0.901042 [t_0.0-0.2=1.1005 t_0.2-0.4=0.9905 t_0.4-0.6=0.8655 t_0.6-0.8=0.6945 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 23.0/188.5 GiB (12.2%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 259/299 (86% done) --- + [Epoch 259][10/66] loss=0.866505 avg=0.897598 VRAM=38.7GiB | 86.4% done | ETA(epoch): 1090s + [Epoch 259][20/66] loss=0.831711 avg=0.864240 VRAM=38.6GiB | 86.4% done | ETA(epoch): 889s + [Epoch 259][30/66] loss=0.899810 avg=0.868243 VRAM=38.7GiB | 86.5% done | ETA(epoch): 695s + [Epoch 259][40/66] loss=0.809634 avg=0.864590 VRAM=38.6GiB | 86.5% done | ETA(epoch): 501s + [Epoch 259][50/66] loss=0.899958 avg=0.864420 VRAM=38.7GiB | 86.6% done | ETA(epoch): 308s + [Epoch 259][60/66] loss=1.011132 avg=0.864439 VRAM=38.6GiB | 86.6% done | ETA(epoch): 116s + Train loss: 0.859838 (1272.5s) ETA: 934min + Val loss: 0.864796 [t_0.0-0.2=1.0496 t_0.2-0.4=0.9899 t_0.4-0.6=0.8695 t_0.6-0.8=0.7302 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 23.5/188.5 GiB (12.5%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 260/299 (87% done) --- + [Epoch 260][10/66] loss=0.921514 avg=0.841827 VRAM=38.7GiB | 86.7% done | ETA(epoch): 1075s + [Epoch 260][20/66] loss=0.805934 avg=0.853031 VRAM=38.6GiB | 86.8% done | ETA(epoch): 884s + [Epoch 260][30/66] loss=0.911751 avg=0.861540 VRAM=38.7GiB | 86.8% done | ETA(epoch): 692s + [Epoch 260][40/66] loss=0.732566 avg=0.864653 VRAM=38.6GiB | 86.9% done | ETA(epoch): 500s + [Epoch 260][50/66] loss=0.859111 avg=0.854990 VRAM=38.7GiB | 86.9% done | ETA(epoch): 308s + [Epoch 260][60/66] loss=0.726051 avg=0.854503 VRAM=38.6GiB | 87.0% done | ETA(epoch): 115s + Train loss: 0.855151 (1268.9s) ETA: 910min + Val loss: 0.821787 [t_0.0-0.2=1.0636 t_0.2-0.4=0.9857 t_0.4-0.6=0.8218 t_0.6-0.8=0.7205 t_0.8-1.0=0.6488] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 261/299 (87% done) --- + [Epoch 261][10/66] loss=0.879369 avg=0.852816 VRAM=38.7GiB | 87.1% done | ETA(epoch): 1076s + [Epoch 261][20/66] loss=0.840933 avg=0.839104 VRAM=38.6GiB | 87.1% done | ETA(epoch): 884s + [Epoch 261][30/66] loss=0.843040 avg=0.853692 VRAM=38.7GiB | 87.2% done | ETA(epoch): 691s + [Epoch 261][40/66] loss=0.914123 avg=0.859230 VRAM=38.6GiB | 87.2% done | ETA(epoch): 499s + [Epoch 261][50/66] loss=0.951717 avg=0.865337 VRAM=38.7GiB | 87.3% done | ETA(epoch): 309s + [Epoch 261][60/66] loss=0.836306 avg=0.863504 VRAM=38.6GiB | 87.3% done | ETA(epoch): 116s + Train loss: 0.860400 (1271.3s) ETA: 887min + Val loss: 0.831433 [t_0.0-0.2=1.0691 t_0.2-0.4=1.0386 t_0.4-0.6=0.8516 t_0.6-0.8=0.6955 t_0.8-1.0=0.6478] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 262/299 (87% done) --- + [Epoch 262][10/66] loss=0.840978 avg=0.870812 VRAM=38.7GiB | 87.4% done | ETA(epoch): 1075s + [Epoch 262][20/66] loss=0.837647 avg=0.873915 VRAM=38.6GiB | 87.4% done | ETA(epoch): 884s + [Epoch 262][30/66] loss=0.714126 avg=0.845724 VRAM=38.7GiB | 87.5% done | ETA(epoch): 692s + [Epoch 262][40/66] loss=0.911270 avg=0.855105 VRAM=38.6GiB | 87.5% done | ETA(epoch): 500s + [Epoch 262][50/66] loss=1.000942 avg=0.860010 VRAM=38.7GiB | 87.6% done | ETA(epoch): 307s + [Epoch 262][60/66] loss=0.862904 avg=0.859640 VRAM=38.6GiB | 87.6% done | ETA(epoch): 115s + Train loss: 0.859443 (1268.2s) ETA: 863min + Val loss: 0.877688 [t_0.0-0.2=1.0794 t_0.2-0.4=1.0241 t_0.4-0.6=0.8563 t_0.6-0.8=0.6974 t_0.8-1.0=0.6585] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 263/299 (88% done) --- + [Epoch 263][10/66] loss=0.851733 avg=0.843263 VRAM=38.7GiB | 87.7% done | ETA(epoch): 1074s + [Epoch 263][20/66] loss=0.792869 avg=0.848642 VRAM=38.6GiB | 87.8% done | ETA(epoch): 882s + [Epoch 263][30/66] loss=0.822719 avg=0.848815 VRAM=38.7GiB | 87.8% done | ETA(epoch): 691s + [Epoch 263][40/66] loss=0.834471 avg=0.853263 VRAM=38.6GiB | 87.9% done | ETA(epoch): 501s + [Epoch 263][50/66] loss=0.809293 avg=0.840416 VRAM=38.7GiB | 87.9% done | ETA(epoch): 308s + [Epoch 263][60/66] loss=0.774475 avg=0.840743 VRAM=38.6GiB | 88.0% done | ETA(epoch): 115s + Train loss: 0.845666 (1269.4s) ETA: 840min + Val loss: 0.889425 [t_0.0-0.2=1.0844 t_0.2-0.4=1.0045 t_0.4-0.6=0.8550 t_0.6-0.8=0.7345 t_0.8-1.0=0.6465] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 264/299 (88% done) --- + [Epoch 264][10/66] loss=0.871289 avg=0.888291 VRAM=38.7GiB | 88.1% done | ETA(epoch): 1075s + [Epoch 264][20/66] loss=0.746918 avg=0.853673 VRAM=38.6GiB | 88.1% done | ETA(epoch): 883s + [Epoch 264][30/66] loss=0.774383 avg=0.846729 VRAM=38.7GiB | 88.2% done | ETA(epoch): 694s + [Epoch 264][40/66] loss=0.864141 avg=0.849112 VRAM=38.6GiB | 88.2% done | ETA(epoch): 501s + [Epoch 264][50/66] loss=0.803138 avg=0.844880 VRAM=38.7GiB | 88.3% done | ETA(epoch): 308s + [Epoch 264][60/66] loss=0.867950 avg=0.843605 VRAM=38.6GiB | 88.3% done | ETA(epoch): 115s + Train loss: 0.843207 (1270.1s) ETA: 816min + Val loss: 0.880174 [t_0.0-0.2=1.0645 t_0.2-0.4=0.9919 t_0.4-0.6=0.8869 t_0.6-0.8=0.7173 t_0.8-1.0=0.6424] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 265/299 (88% done) --- + [Epoch 265][10/66] loss=0.877253 avg=0.802285 VRAM=38.7GiB | 88.4% done | ETA(epoch): 1078s + [Epoch 265][20/66] loss=0.781823 avg=0.846104 VRAM=38.6GiB | 88.4% done | ETA(epoch): 885s + [Epoch 265][30/66] loss=0.795703 avg=0.830161 VRAM=38.7GiB | 88.5% done | ETA(epoch): 692s + [Epoch 265][40/66] loss=0.865242 avg=0.837912 VRAM=38.6GiB | 88.5% done | ETA(epoch): 500s + [Epoch 265][50/66] loss=0.825094 avg=0.842559 VRAM=38.7GiB | 88.6% done | ETA(epoch): 308s + [Epoch 265][60/66] loss=0.903953 avg=0.847875 VRAM=38.6GiB | 88.6% done | ETA(epoch): 115s + Train loss: 0.846776 (1268.6s) ETA: 793min + Val loss: 0.816033 [t_0.0-0.2=1.0497 t_0.2-0.4=0.9750 t_0.4-0.6=0.8451 t_0.6-0.8=0.7213 t_0.8-1.0=0.6158] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 24.1/188.5 GiB (12.8%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 266/299 (89% done) --- + [Epoch 266][10/66] loss=0.858369 avg=0.827048 VRAM=38.7GiB | 88.7% done | ETA(epoch): 1074s + [Epoch 266][20/66] loss=1.025254 avg=0.840994 VRAM=38.6GiB | 88.8% done | ETA(epoch): 884s + [Epoch 266][30/66] loss=0.831782 avg=0.840449 VRAM=38.7GiB | 88.8% done | ETA(epoch): 692s + [Epoch 266][40/66] loss=0.894611 avg=0.847680 VRAM=38.6GiB | 88.9% done | ETA(epoch): 500s + [Epoch 266][50/66] loss=0.777001 avg=0.843925 VRAM=38.7GiB | 88.9% done | ETA(epoch): 307s + [Epoch 266][60/66] loss=0.863950 avg=0.847493 VRAM=38.6GiB | 89.0% done | ETA(epoch): 115s + Train loss: 0.851658 (1268.1s) ETA: 770min + Val loss: 0.820475 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9836 t_0.4-0.6=0.8123 t_0.6-0.8=0.7084 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 267/299 (89% done) --- + [Epoch 267][10/66] loss=0.880332 avg=0.872434 VRAM=38.7GiB | 89.1% done | ETA(epoch): 1075s + [Epoch 267][20/66] loss=0.796683 avg=0.857477 VRAM=38.6GiB | 89.1% done | ETA(epoch): 883s + [Epoch 267][30/66] loss=0.889493 avg=0.863686 VRAM=38.7GiB | 89.2% done | ETA(epoch): 691s + [Epoch 267][40/66] loss=0.714275 avg=0.858819 VRAM=38.6GiB | 89.2% done | ETA(epoch): 499s + [Epoch 267][50/66] loss=0.892087 avg=0.856967 VRAM=38.7GiB | 89.3% done | ETA(epoch): 307s + [Epoch 267][60/66] loss=0.893090 avg=0.860385 VRAM=38.6GiB | 89.3% done | ETA(epoch): 115s + Train loss: 0.861463 (1268.2s) ETA: 746min + Val loss: 0.887458 [t_0.0-0.2=1.0915 t_0.2-0.4=0.9805 t_0.4-0.6=0.8641 t_0.6-0.8=0.6945 t_0.8-1.0=0.6393] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 268/299 (89% done) --- + [Epoch 268][10/66] loss=0.830330 avg=0.881259 VRAM=38.7GiB | 89.4% done | ETA(epoch): 1075s + [Epoch 268][20/66] loss=0.912958 avg=0.851650 VRAM=38.6GiB | 89.4% done | ETA(epoch): 883s + [Epoch 268][30/66] loss=0.828455 avg=0.856869 VRAM=38.7GiB | 89.5% done | ETA(epoch): 692s + [Epoch 268][40/66] loss=0.870790 avg=0.865022 VRAM=38.6GiB | 89.5% done | ETA(epoch): 499s + [Epoch 268][50/66] loss=0.940972 avg=0.864713 VRAM=38.7GiB | 89.6% done | ETA(epoch): 307s + [Epoch 268][60/66] loss=0.760019 avg=0.863447 VRAM=38.6GiB | 89.6% done | ETA(epoch): 115s + Train loss: 0.864374 (1267.7s) ETA: 723min + Val loss: 0.874273 [t_0.0-0.2=1.0617 t_0.2-0.4=1.0062 t_0.4-0.6=0.8645 t_0.6-0.8=0.7095 t_0.8-1.0=0.6155] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 269/299 (90% done) --- + [Epoch 269][10/66] loss=0.910637 avg=0.897276 VRAM=38.7GiB | 89.7% done | ETA(epoch): 1075s + [Epoch 269][20/66] loss=0.883642 avg=0.875626 VRAM=38.6GiB | 89.8% done | ETA(epoch): 883s + [Epoch 269][30/66] loss=0.839080 avg=0.859921 VRAM=38.7GiB | 89.8% done | ETA(epoch): 691s + [Epoch 269][40/66] loss=0.873243 avg=0.859926 VRAM=38.6GiB | 89.9% done | ETA(epoch): 499s + [Epoch 269][50/66] loss=0.890825 avg=0.857572 VRAM=38.7GiB | 89.9% done | ETA(epoch): 308s + [Epoch 269][60/66] loss=0.762237 avg=0.852493 VRAM=38.6GiB | 90.0% done | ETA(epoch): 115s + [MilestoneVis] train_0 step 17819 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 17819 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 17819 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 17819 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 17819 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 17819 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 17819 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 17819 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 17819 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 17819 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.852377 (2311.1s) ETA: 701min + Val loss: 0.835930 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9765 t_0.4-0.6=0.8487 t_0.6-0.8=0.7105 t_0.8-1.0=0.6254] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 270/299 (90% done) --- + [Epoch 270][10/66] loss=0.803457 avg=0.856387 VRAM=38.7GiB | 90.1% done | ETA(epoch): 1076s + [Epoch 270][20/66] loss=0.794087 avg=0.845150 VRAM=38.6GiB | 90.1% done | ETA(epoch): 884s + [Epoch 270][30/66] loss=0.931698 avg=0.861029 VRAM=38.7GiB | 90.2% done | ETA(epoch): 692s + [Epoch 270][40/66] loss=0.875246 avg=0.856395 VRAM=38.6GiB | 90.2% done | ETA(epoch): 500s + [Epoch 270][50/66] loss=0.749314 avg=0.847630 VRAM=38.7GiB | 90.3% done | ETA(epoch): 308s + [Epoch 270][60/66] loss=0.879891 avg=0.852052 VRAM=38.6GiB | 90.3% done | ETA(epoch): 115s + Train loss: 0.848097 (1268.4s) ETA: 678min + Val loss: 0.851113 [t_0.0-0.2=1.0584 t_0.2-0.4=1.0264 t_0.4-0.6=0.8691 t_0.6-0.8=0.7360 t_0.8-1.0=0.6244] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 271/299 (90% done) --- + [Epoch 271][10/66] loss=0.800283 avg=0.870540 VRAM=38.7GiB | 90.4% done | ETA(epoch): 1075s + [Epoch 271][20/66] loss=0.882211 avg=0.861349 VRAM=38.6GiB | 90.4% done | ETA(epoch): 884s + [Epoch 271][30/66] loss=0.759678 avg=0.854100 VRAM=38.7GiB | 90.5% done | ETA(epoch): 692s + [Epoch 271][40/66] loss=0.869031 avg=0.854120 VRAM=38.6GiB | 90.5% done | ETA(epoch): 500s + [Epoch 271][50/66] loss=0.826995 avg=0.854474 VRAM=38.7GiB | 90.6% done | ETA(epoch): 307s + [Epoch 271][60/66] loss=0.847703 avg=0.856223 VRAM=38.6GiB | 90.6% done | ETA(epoch): 115s + Train loss: 0.856576 (1268.7s) ETA: 654min + Val loss: 0.841879 [t_0.0-0.2=1.0618 t_0.2-0.4=1.0082 t_0.4-0.6=0.8176 t_0.6-0.8=0.7336 t_0.8-1.0=0.6433] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 272/299 (91% done) --- + [Epoch 272][10/66] loss=0.720875 avg=0.831022 VRAM=38.7GiB | 90.7% done | ETA(epoch): 1075s + [Epoch 272][20/66] loss=0.834659 avg=0.853731 VRAM=38.6GiB | 90.8% done | ETA(epoch): 883s + [Epoch 272][30/66] loss=0.968291 avg=0.866415 VRAM=38.7GiB | 90.8% done | ETA(epoch): 694s + [Epoch 272][40/66] loss=0.969220 avg=0.872872 VRAM=38.6GiB | 90.9% done | ETA(epoch): 501s + [Epoch 272][50/66] loss=0.824965 avg=0.869128 VRAM=38.7GiB | 90.9% done | ETA(epoch): 308s + [Epoch 272][60/66] loss=0.735578 avg=0.866502 VRAM=38.6GiB | 91.0% done | ETA(epoch): 115s + Train loss: 0.873842 (1270.2s) ETA: 631min + Val loss: 0.810344 [t_0.0-0.2=1.0434 t_0.2-0.4=0.9955 t_0.4-0.6=0.8576 t_0.6-0.8=0.6798 t_0.8-1.0=0.6438] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 24.3/188.5 GiB (12.9%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 273/299 (91% done) --- + [Epoch 273][10/66] loss=0.898277 avg=0.863984 VRAM=38.7GiB | 91.1% done | ETA(epoch): 1075s + [Epoch 273][20/66] loss=0.952804 avg=0.859807 VRAM=38.6GiB | 91.1% done | ETA(epoch): 886s + [Epoch 273][30/66] loss=0.837125 avg=0.851133 VRAM=38.7GiB | 91.2% done | ETA(epoch): 693s + [Epoch 273][40/66] loss=0.811442 avg=0.853741 VRAM=38.6GiB | 91.2% done | ETA(epoch): 500s + [Epoch 273][50/66] loss=0.967587 avg=0.855065 VRAM=38.7GiB | 91.3% done | ETA(epoch): 308s + [Epoch 273][60/66] loss=0.807441 avg=0.856407 VRAM=38.6GiB | 91.3% done | ETA(epoch): 115s + Train loss: 0.856018 (1269.0s) ETA: 608min + Val loss: 0.873540 [t_0.0-0.2=1.0699 t_0.2-0.4=0.9870 t_0.4-0.6=0.8664 t_0.6-0.8=0.7170 t_0.8-1.0=0.6369] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 23.2/188.5 GiB (12.3%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 274/299 (91% done) --- + [Epoch 274][10/66] loss=0.910890 avg=0.829592 VRAM=38.7GiB | 91.4% done | ETA(epoch): 1074s + [Epoch 274][20/66] loss=0.783227 avg=0.839866 VRAM=38.6GiB | 91.4% done | ETA(epoch): 883s + [Epoch 274][30/66] loss=0.838346 avg=0.835261 VRAM=38.7GiB | 91.5% done | ETA(epoch): 691s + [Epoch 274][40/66] loss=0.941285 avg=0.848166 VRAM=38.6GiB | 91.5% done | ETA(epoch): 499s + [Epoch 274][50/66] loss=0.728190 avg=0.844830 VRAM=38.7GiB | 91.6% done | ETA(epoch): 307s + [Epoch 274][60/66] loss=0.808683 avg=0.840785 VRAM=38.6GiB | 91.6% done | ETA(epoch): 115s + Train loss: 0.840428 (1267.6s) ETA: 584min + Val loss: 0.868347 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9929 t_0.4-0.6=0.8653 t_0.6-0.8=0.7126 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 275/299 (92% done) --- + [Epoch 275][10/66] loss=0.872273 avg=0.848095 VRAM=38.7GiB | 91.7% done | ETA(epoch): 1074s + [Epoch 275][20/66] loss=0.927693 avg=0.849619 VRAM=38.6GiB | 91.8% done | ETA(epoch): 883s + [Epoch 275][30/66] loss=0.870334 avg=0.845172 VRAM=38.7GiB | 91.8% done | ETA(epoch): 691s + [Epoch 275][40/66] loss=0.881065 avg=0.851035 VRAM=38.6GiB | 91.9% done | ETA(epoch): 499s + [Epoch 275][50/66] loss=0.792246 avg=0.850652 VRAM=38.7GiB | 91.9% done | ETA(epoch): 307s + [Epoch 275][60/66] loss=0.939721 avg=0.849584 VRAM=38.6GiB | 92.0% done | ETA(epoch): 115s + Train loss: 0.850418 (1268.4s) ETA: 561min + Val loss: 0.847669 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9639 t_0.4-0.6=0.8770 t_0.6-0.8=0.7109 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 276/299 (92% done) --- + [Epoch 276][10/66] loss=0.860525 avg=0.853552 VRAM=38.7GiB | 92.1% done | ETA(epoch): 1074s + [Epoch 276][20/66] loss=0.854595 avg=0.847901 VRAM=38.6GiB | 92.1% done | ETA(epoch): 883s + [Epoch 276][30/66] loss=0.849901 avg=0.850350 VRAM=38.7GiB | 92.2% done | ETA(epoch): 692s + [Epoch 276][40/66] loss=0.797401 avg=0.845602 VRAM=38.6GiB | 92.2% done | ETA(epoch): 500s + [Epoch 276][50/66] loss=0.807652 avg=0.842622 VRAM=38.7GiB | 92.3% done | ETA(epoch): 308s + [Epoch 276][60/66] loss=0.805451 avg=0.837284 VRAM=38.6GiB | 92.3% done | ETA(epoch): 115s + Train loss: 0.840487 (1268.6s) ETA: 537min + Val loss: 0.862763 [t_0.0-0.2=1.0740 t_0.2-0.4=1.0149 t_0.4-0.6=0.8335 t_0.6-0.8=0.7264 t_0.8-1.0=0.6405] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 277/299 (92% done) --- + [Epoch 277][10/66] loss=0.864155 avg=0.829124 VRAM=38.7GiB | 92.4% done | ETA(epoch): 1074s + [Epoch 277][20/66] loss=0.995462 avg=0.851279 VRAM=38.6GiB | 92.4% done | ETA(epoch): 883s + [Epoch 277][30/66] loss=0.892569 avg=0.850602 VRAM=38.7GiB | 92.5% done | ETA(epoch): 691s + [Epoch 277][40/66] loss=0.856896 avg=0.858217 VRAM=38.6GiB | 92.5% done | ETA(epoch): 499s + [Epoch 277][50/66] loss=1.018945 avg=0.858600 VRAM=38.7GiB | 92.6% done | ETA(epoch): 308s + [Epoch 277][60/66] loss=0.942439 avg=0.857772 VRAM=38.6GiB | 92.6% done | ETA(epoch): 116s + Train loss: 0.858138 (1271.4s) ETA: 514min + Val loss: 0.878104 [t_0.0-0.2=1.0728 t_0.2-0.4=0.9803 t_0.4-0.6=0.8301 t_0.6-0.8=0.7315 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 278/299 (93% done) --- + [Epoch 278][10/66] loss=0.791861 avg=0.865217 VRAM=38.7GiB | 92.7% done | ETA(epoch): 1075s + [Epoch 278][20/66] loss=0.846583 avg=0.868748 VRAM=38.6GiB | 92.8% done | ETA(epoch): 883s + [Epoch 278][30/66] loss=0.808265 avg=0.850146 VRAM=38.7GiB | 92.8% done | ETA(epoch): 691s + [Epoch 278][40/66] loss=0.919343 avg=0.849991 VRAM=38.6GiB | 92.9% done | ETA(epoch): 499s + [Epoch 278][50/66] loss=0.814867 avg=0.855439 VRAM=38.7GiB | 92.9% done | ETA(epoch): 308s + [Epoch 278][60/66] loss=0.818293 avg=0.852454 VRAM=38.6GiB | 93.0% done | ETA(epoch): 115s + Train loss: 0.853288 (1269.7s) ETA: 491min + Val loss: 0.864988 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9882 t_0.4-0.6=0.8290 t_0.6-0.8=0.7296 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 279/299 (93% done) --- + [Epoch 279][10/66] loss=0.792819 avg=0.842444 VRAM=38.7GiB | 93.1% done | ETA(epoch): 1077s + [Epoch 279][20/66] loss=0.758525 avg=0.834814 VRAM=38.6GiB | 93.1% done | ETA(epoch): 884s + [Epoch 279][30/66] loss=0.758990 avg=0.837574 VRAM=38.7GiB | 93.2% done | ETA(epoch): 693s + [Epoch 279][40/66] loss=0.973292 avg=0.849792 VRAM=38.6GiB | 93.2% done | ETA(epoch): 503s + [Epoch 279][50/66] loss=0.803902 avg=0.849065 VRAM=38.7GiB | 93.3% done | ETA(epoch): 309s + [Epoch 279][60/66] loss=0.722092 avg=0.846982 VRAM=38.6GiB | 93.3% done | ETA(epoch): 116s + Train loss: 0.852049 (1273.7s) ETA: 467min + Val loss: 0.828790 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0224 t_0.4-0.6=0.8580 t_0.6-0.8=0.6830 t_0.8-1.0=0.6366] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 280/299 (93% done) --- + [Epoch 280][10/66] loss=0.951036 avg=0.855300 VRAM=38.7GiB | 93.4% done | ETA(epoch): 1074s + [Epoch 280][20/66] loss=0.775690 avg=0.862887 VRAM=38.6GiB | 93.4% done | ETA(epoch): 884s + [Epoch 280][30/66] loss=0.756618 avg=0.849277 VRAM=38.7GiB | 93.5% done | ETA(epoch): 692s + [Epoch 280][40/66] loss=0.696419 avg=0.843371 VRAM=38.6GiB | 93.5% done | ETA(epoch): 500s + [Epoch 280][50/66] loss=0.933570 avg=0.846161 VRAM=38.7GiB | 93.6% done | ETA(epoch): 308s + [Epoch 280][60/66] loss=0.878570 avg=0.847373 VRAM=38.6GiB | 93.6% done | ETA(epoch): 115s + Train loss: 0.852131 (1269.3s) ETA: 444min + Val loss: 0.869870 [t_0.0-0.2=1.0837 t_0.2-0.4=1.0197 t_0.4-0.6=0.8159 t_0.6-0.8=0.6850 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 281/299 (94% done) --- + [Epoch 281][10/66] loss=0.844327 avg=0.848540 VRAM=38.7GiB | 93.7% done | ETA(epoch): 1076s + [Epoch 281][20/66] loss=0.893677 avg=0.860447 VRAM=38.6GiB | 93.8% done | ETA(epoch): 883s + [Epoch 281][30/66] loss=0.834483 avg=0.853282 VRAM=38.7GiB | 93.8% done | ETA(epoch): 692s + [Epoch 281][40/66] loss=0.756821 avg=0.854126 VRAM=38.6GiB | 93.9% done | ETA(epoch): 500s + [Epoch 281][50/66] loss=0.963797 avg=0.855446 VRAM=38.7GiB | 93.9% done | ETA(epoch): 308s + [Epoch 281][60/66] loss=1.001598 avg=0.860530 VRAM=38.6GiB | 94.0% done | ETA(epoch): 115s + Train loss: 0.857672 (1269.1s) ETA: 420min + Val loss: 0.799002 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9908 t_0.4-0.6=0.8391 t_0.6-0.8=0.6995 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0281 (BEST) + Deleted old checkpoint: checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 282/299 (94% done) --- + [Epoch 282][10/66] loss=0.775729 avg=0.815379 VRAM=38.7GiB | 94.1% done | ETA(epoch): 1077s + [Epoch 282][20/66] loss=0.933401 avg=0.868789 VRAM=38.6GiB | 94.1% done | ETA(epoch): 884s + [Epoch 282][30/66] loss=1.029049 avg=0.877673 VRAM=38.7GiB | 94.2% done | ETA(epoch): 692s + [Epoch 282][40/66] loss=0.856307 avg=0.861603 VRAM=38.6GiB | 94.2% done | ETA(epoch): 500s + [Epoch 282][50/66] loss=0.834165 avg=0.862801 VRAM=38.7GiB | 94.3% done | ETA(epoch): 308s + [Epoch 282][60/66] loss=0.914447 avg=0.854024 VRAM=38.6GiB | 94.3% done | ETA(epoch): 115s + Train loss: 0.853460 (1268.7s) ETA: 397min + Val loss: 0.813957 [t_0.0-0.2=1.0594 t_0.2-0.4=1.0022 t_0.4-0.6=0.8432 t_0.6-0.8=0.7028 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 283/299 (94% done) --- + [Epoch 283][10/66] loss=0.900681 avg=0.855126 VRAM=38.7GiB | 94.4% done | ETA(epoch): 1075s + [Epoch 283][20/66] loss=0.892057 avg=0.866753 VRAM=38.6GiB | 94.4% done | ETA(epoch): 884s + [Epoch 283][30/66] loss=0.789576 avg=0.858098 VRAM=38.7GiB | 94.5% done | ETA(epoch): 691s + [Epoch 283][40/66] loss=0.768705 avg=0.856614 VRAM=38.6GiB | 94.5% done | ETA(epoch): 500s + [Epoch 283][50/66] loss=0.924975 avg=0.856054 VRAM=38.7GiB | 94.6% done | ETA(epoch): 308s + [Epoch 283][60/66] loss=0.791348 avg=0.854062 VRAM=38.6GiB | 94.6% done | ETA(epoch): 115s + Train loss: 0.851124 (1269.0s) ETA: 374min + Val loss: 0.843419 [t_0.0-0.2=1.0766 t_0.2-0.4=1.0010 t_0.4-0.6=0.8398 t_0.6-0.8=0.6939 t_0.8-1.0=0.6264] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 284/299 (95% done) --- + [Epoch 284][10/66] loss=0.939206 avg=0.870777 VRAM=38.7GiB | 94.7% done | ETA(epoch): 1074s + [Epoch 284][20/66] loss=0.734373 avg=0.860316 VRAM=38.6GiB | 94.8% done | ETA(epoch): 883s + [Epoch 284][30/66] loss=0.916294 avg=0.864278 VRAM=38.7GiB | 94.8% done | ETA(epoch): 691s + [Epoch 284][40/66] loss=0.812669 avg=0.857513 VRAM=38.6GiB | 94.9% done | ETA(epoch): 499s + [Epoch 284][50/66] loss=0.696863 avg=0.853931 VRAM=38.7GiB | 94.9% done | ETA(epoch): 307s + [Epoch 284][60/66] loss=0.826561 avg=0.848710 VRAM=38.6GiB | 95.0% done | ETA(epoch): 116s + Train loss: 0.845028 (1270.9s) ETA: 350min + Val loss: 0.842229 [t_0.0-0.2=1.0675 t_0.2-0.4=1.0157 t_0.4-0.6=0.8341 t_0.6-0.8=0.7116 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0284 +[MEM @ epoch 284 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 285/299 (95% done) --- + [Epoch 285][10/66] loss=0.812288 avg=0.851236 VRAM=38.7GiB | 95.1% done | ETA(epoch): 1075s + [Epoch 285][20/66] loss=0.952250 avg=0.877150 VRAM=38.6GiB | 95.1% done | ETA(epoch): 883s + [Epoch 285][30/66] loss=0.777706 avg=0.868910 VRAM=38.7GiB | 95.2% done | ETA(epoch): 691s + [Epoch 285][40/66] loss=0.831796 avg=0.869113 VRAM=38.6GiB | 95.2% done | ETA(epoch): 500s + [Epoch 285][50/66] loss=0.889004 avg=0.866655 VRAM=38.7GiB | 95.3% done | ETA(epoch): 308s + [Epoch 285][60/66] loss=0.857162 avg=0.856834 VRAM=38.6GiB | 95.3% done | ETA(epoch): 115s + Train loss: 0.855138 (1270.0s) ETA: 327min + Val loss: 0.802088 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9841 t_0.4-0.6=0.8413 t_0.6-0.8=0.7060 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 286/299 (95% done) --- + [Epoch 286][10/66] loss=0.888617 avg=0.870018 VRAM=38.7GiB | 95.4% done | ETA(epoch): 1075s + [Epoch 286][20/66] loss=1.047630 avg=0.888208 VRAM=38.6GiB | 95.4% done | ETA(epoch): 883s + [Epoch 286][30/66] loss=0.891484 avg=0.884606 VRAM=38.7GiB | 95.5% done | ETA(epoch): 691s + [Epoch 286][40/66] loss=0.933830 avg=0.873252 VRAM=38.6GiB | 95.5% done | ETA(epoch): 502s + [Epoch 286][50/66] loss=0.873675 avg=0.871920 VRAM=38.7GiB | 95.6% done | ETA(epoch): 309s + [Epoch 286][60/66] loss=0.807618 avg=0.870212 VRAM=38.6GiB | 95.6% done | ETA(epoch): 116s + Train loss: 0.869569 (1272.2s) ETA: 303min + Val loss: 0.822347 [t_0.0-0.2=1.0874 t_0.2-0.4=1.0082 t_0.4-0.6=0.8717 t_0.6-0.8=0.7068 t_0.8-1.0=0.6436] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 287/299 (96% done) --- + [Epoch 287][10/66] loss=0.838554 avg=0.871867 VRAM=38.7GiB | 95.7% done | ETA(epoch): 1077s + [Epoch 287][20/66] loss=0.881357 avg=0.869612 VRAM=38.6GiB | 95.8% done | ETA(epoch): 885s + [Epoch 287][30/66] loss=0.855981 avg=0.869558 VRAM=38.7GiB | 95.8% done | ETA(epoch): 692s + [Epoch 287][40/66] loss=0.740053 avg=0.850899 VRAM=38.6GiB | 95.9% done | ETA(epoch): 500s + [Epoch 287][50/66] loss=0.898729 avg=0.849310 VRAM=38.7GiB | 95.9% done | ETA(epoch): 308s + [Epoch 287][60/66] loss=0.874382 avg=0.851165 VRAM=38.6GiB | 96.0% done | ETA(epoch): 115s + Train loss: 0.851352 (1269.3s) ETA: 280min + Val loss: 0.844504 [t_0.0-0.2=1.0899 t_0.2-0.4=0.9939 t_0.4-0.6=0.8484 t_0.6-0.8=0.6926 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 288/299 (96% done) --- + [Epoch 288][10/66] loss=0.797904 avg=0.809958 VRAM=38.7GiB | 96.1% done | ETA(epoch): 1077s + [Epoch 288][20/66] loss=0.996388 avg=0.871621 VRAM=38.6GiB | 96.1% done | ETA(epoch): 885s + [Epoch 288][30/66] loss=0.835702 avg=0.853681 VRAM=38.7GiB | 96.2% done | ETA(epoch): 692s + [Epoch 288][40/66] loss=0.904861 avg=0.851982 VRAM=38.6GiB | 96.2% done | ETA(epoch): 500s + [Epoch 288][50/66] loss=0.998503 avg=0.864906 VRAM=38.7GiB | 96.3% done | ETA(epoch): 308s + [Epoch 288][60/66] loss=0.988966 avg=0.865949 VRAM=38.6GiB | 96.3% done | ETA(epoch): 115s + Train loss: 0.863186 (1269.6s) ETA: 257min + Val loss: 0.843094 [t_0.0-0.2=1.0903 t_0.2-0.4=1.0070 t_0.4-0.6=0.8179 t_0.6-0.8=0.7090 t_0.8-1.0=0.6379] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 289/299 (96% done) --- + [Epoch 289][10/66] loss=0.931355 avg=0.838301 VRAM=38.7GiB | 96.4% done | ETA(epoch): 1075s + [Epoch 289][20/66] loss=0.768459 avg=0.826503 VRAM=38.6GiB | 96.4% done | ETA(epoch): 888s + [Epoch 289][30/66] loss=0.766940 avg=0.822118 VRAM=38.7GiB | 96.5% done | ETA(epoch): 694s + [Epoch 289][40/66] loss=0.860494 avg=0.829758 VRAM=38.6GiB | 96.5% done | ETA(epoch): 501s + [Epoch 289][50/66] loss=0.814501 avg=0.832148 VRAM=38.7GiB | 96.6% done | ETA(epoch): 308s + [Epoch 289][60/66] loss=0.935802 avg=0.828691 VRAM=38.6GiB | 96.6% done | ETA(epoch): 115s + Train loss: 0.831646 (1270.3s) ETA: 233min + Val loss: 0.837234 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9721 t_0.4-0.6=0.8824 t_0.6-0.8=0.7278 t_0.8-1.0=0.6353] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 290/299 (97% done) --- + [Epoch 290][10/66] loss=0.748409 avg=0.869828 VRAM=38.7GiB | 96.7% done | ETA(epoch): 1077s + [Epoch 290][20/66] loss=0.936431 avg=0.860404 VRAM=38.6GiB | 96.8% done | ETA(epoch): 885s + [Epoch 290][30/66] loss=0.858570 avg=0.859130 VRAM=38.7GiB | 96.8% done | ETA(epoch): 692s + [Epoch 290][40/66] loss=0.764232 avg=0.849396 VRAM=38.6GiB | 96.9% done | ETA(epoch): 500s + [Epoch 290][50/66] loss=0.878094 avg=0.849867 VRAM=38.7GiB | 96.9% done | ETA(epoch): 308s + [Epoch 290][60/66] loss=0.879272 avg=0.852673 VRAM=38.6GiB | 97.0% done | ETA(epoch): 115s + Train loss: 0.858888 (1270.0s) ETA: 210min + Val loss: 0.851807 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9765 t_0.4-0.6=0.8690 t_0.6-0.8=0.7113 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 291/299 (97% done) --- + [Epoch 291][10/66] loss=0.807200 avg=0.834368 VRAM=38.7GiB | 97.1% done | ETA(epoch): 1078s + [Epoch 291][20/66] loss=0.948627 avg=0.861018 VRAM=38.6GiB | 97.1% done | ETA(epoch): 884s + [Epoch 291][30/66] loss=0.830414 avg=0.846439 VRAM=38.7GiB | 97.2% done | ETA(epoch): 692s + [Epoch 291][40/66] loss=0.913777 avg=0.847033 VRAM=38.6GiB | 97.2% done | ETA(epoch): 500s + [Epoch 291][50/66] loss=0.817239 avg=0.848599 VRAM=38.7GiB | 97.3% done | ETA(epoch): 308s + [Epoch 291][60/66] loss=0.952853 avg=0.847024 VRAM=38.6GiB | 97.3% done | ETA(epoch): 115s + Train loss: 0.849714 (1270.3s) ETA: 187min + Val loss: 0.855713 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9872 t_0.4-0.6=0.8675 t_0.6-0.8=0.7062 t_0.8-1.0=0.6296] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 292/299 (97% done) --- + [Epoch 292][10/66] loss=0.914074 avg=0.812109 VRAM=38.7GiB | 97.4% done | ETA(epoch): 1078s + [Epoch 292][20/66] loss=0.886055 avg=0.827928 VRAM=38.6GiB | 97.4% done | ETA(epoch): 885s + [Epoch 292][30/66] loss=0.809664 avg=0.828845 VRAM=38.7GiB | 97.5% done | ETA(epoch): 692s + [Epoch 292][40/66] loss=0.975720 avg=0.838314 VRAM=38.6GiB | 97.5% done | ETA(epoch): 500s + [Epoch 292][50/66] loss=0.963280 avg=0.849924 VRAM=38.7GiB | 97.6% done | ETA(epoch): 307s + [Epoch 292][60/66] loss=0.837118 avg=0.853218 VRAM=38.6GiB | 97.6% done | ETA(epoch): 115s + Train loss: 0.858322 (1270.6s) ETA: 163min + Val loss: 0.863528 [t_0.0-0.2=1.0651 t_0.2-0.4=0.9898 t_0.4-0.6=0.8566 t_0.6-0.8=0.7227 t_0.8-1.0=0.6263] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 293/299 (98% done) --- + [Epoch 293][10/66] loss=0.739120 avg=0.830920 VRAM=38.7GiB | 97.7% done | ETA(epoch): 1078s + [Epoch 293][20/66] loss=0.857761 avg=0.839210 VRAM=38.6GiB | 97.8% done | ETA(epoch): 885s + [Epoch 293][30/66] loss=0.799422 avg=0.833658 VRAM=38.7GiB | 97.8% done | ETA(epoch): 692s + [Epoch 293][40/66] loss=0.925976 avg=0.845420 VRAM=38.6GiB | 97.9% done | ETA(epoch): 500s + [Epoch 293][50/66] loss=0.885847 avg=0.845715 VRAM=38.7GiB | 97.9% done | ETA(epoch): 308s + [Epoch 293][60/66] loss=0.861886 avg=0.847683 VRAM=38.6GiB | 98.0% done | ETA(epoch): 116s + Train loss: 0.844823 (1271.6s) ETA: 140min + Val loss: 0.840480 [t_0.0-0.2=1.0759 t_0.2-0.4=0.9710 t_0.4-0.6=0.8463 t_0.6-0.8=0.7060 t_0.8-1.0=0.6492] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 23.7/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 294/299 (98% done) --- + [Epoch 294][10/66] loss=0.791193 avg=0.856793 VRAM=38.7GiB | 98.1% done | ETA(epoch): 1077s + [Epoch 294][20/66] loss=0.920695 avg=0.882460 VRAM=38.6GiB | 98.1% done | ETA(epoch): 885s + [Epoch 294][30/66] loss=0.833776 avg=0.865913 VRAM=38.7GiB | 98.2% done | ETA(epoch): 693s + [Epoch 294][40/66] loss=0.840918 avg=0.857731 VRAM=38.6GiB | 98.2% done | ETA(epoch): 502s + [Epoch 294][50/66] loss=0.782675 avg=0.856124 VRAM=38.7GiB | 98.3% done | ETA(epoch): 309s + [Epoch 294][60/66] loss=0.914813 avg=0.849559 VRAM=38.6GiB | 98.3% done | ETA(epoch): 116s + Train loss: 0.851471 (1272.4s) ETA: 117min + Val loss: 0.844193 [t_0.0-0.2=1.0906 t_0.2-0.4=1.0236 t_0.4-0.6=0.8394 t_0.6-0.8=0.7103 t_0.8-1.0=0.6427] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 295/299 (98% done) --- + [Epoch 295][10/66] loss=0.800827 avg=0.867514 VRAM=38.7GiB | 98.4% done | ETA(epoch): 1076s + [Epoch 295][20/66] loss=0.911930 avg=0.853573 VRAM=38.6GiB | 98.4% done | ETA(epoch): 885s + [Epoch 295][30/66] loss=0.913446 avg=0.859173 VRAM=38.7GiB | 98.5% done | ETA(epoch): 693s + [Epoch 295][40/66] loss=0.800820 avg=0.859394 VRAM=38.6GiB | 98.5% done | ETA(epoch): 501s + [Epoch 295][50/66] loss=0.922970 avg=0.864127 VRAM=38.7GiB | 98.6% done | ETA(epoch): 308s + [Epoch 295][60/66] loss=0.917433 avg=0.865966 VRAM=38.6GiB | 98.6% done | ETA(epoch): 115s + Train loss: 0.861759 (1270.4s) ETA: 93min + Val loss: 0.858091 [t_0.0-0.2=1.0747 t_0.2-0.4=0.9701 t_0.4-0.6=0.8451 t_0.6-0.8=0.7172 t_0.8-1.0=0.6267] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 296/299 (99% done) --- + [Epoch 296][10/66] loss=0.927037 avg=0.852718 VRAM=38.7GiB | 98.7% done | ETA(epoch): 1078s + [Epoch 296][20/66] loss=0.772143 avg=0.862788 VRAM=38.6GiB | 98.8% done | ETA(epoch): 885s + [Epoch 296][30/66] loss=0.903084 avg=0.861819 VRAM=38.7GiB | 98.8% done | ETA(epoch): 693s + [Epoch 296][40/66] loss=0.848512 avg=0.851894 VRAM=38.6GiB | 98.9% done | ETA(epoch): 500s + [Epoch 296][50/66] loss=0.846683 avg=0.845283 VRAM=38.7GiB | 98.9% done | ETA(epoch): 308s + [Epoch 296][60/66] loss=0.880367 avg=0.846000 VRAM=38.6GiB | 99.0% done | ETA(epoch): 115s + Train loss: 0.849050 (1269.2s) ETA: 70min + Val loss: 0.863780 [t_0.0-0.2=1.0684 t_0.2-0.4=0.9881 t_0.4-0.6=0.8397 t_0.6-0.8=0.7146 t_0.8-1.0=0.6561] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 297/299 (99% done) --- + [Epoch 297][10/66] loss=0.871498 avg=0.853827 VRAM=38.7GiB | 99.1% done | ETA(epoch): 1089s + [Epoch 297][20/66] loss=0.887795 avg=0.870565 VRAM=38.6GiB | 99.1% done | ETA(epoch): 890s + [Epoch 297][30/66] loss=0.768476 avg=0.870272 VRAM=38.7GiB | 99.2% done | ETA(epoch): 695s + [Epoch 297][40/66] loss=0.924783 avg=0.867030 VRAM=38.6GiB | 99.2% done | ETA(epoch): 501s + [Epoch 297][50/66] loss=0.866824 avg=0.866242 VRAM=38.7GiB | 99.3% done | ETA(epoch): 308s + [Epoch 297][60/66] loss=0.811213 avg=0.862817 VRAM=38.6GiB | 99.3% done | ETA(epoch): 116s + Train loss: 0.863200 (1271.1s) ETA: 47min + Val loss: 0.858209 [t_0.0-0.2=1.0774 t_0.2-0.4=1.0044 t_0.4-0.6=0.8309 t_0.6-0.8=0.7213 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 24.0/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 298/299 (99% done) --- + [Epoch 298][10/66] loss=0.871311 avg=0.887356 VRAM=38.7GiB | 99.4% done | ETA(epoch): 1076s + [Epoch 298][20/66] loss=0.700200 avg=0.856846 VRAM=38.6GiB | 99.4% done | ETA(epoch): 884s + [Epoch 298][30/66] loss=0.785431 avg=0.841583 VRAM=38.7GiB | 99.5% done | ETA(epoch): 692s + [Epoch 298][40/66] loss=0.936711 avg=0.836163 VRAM=38.6GiB | 99.5% done | ETA(epoch): 500s + [Epoch 298][50/66] loss=0.823456 avg=0.834777 VRAM=38.7GiB | 99.6% done | ETA(epoch): 308s + [Epoch 298][60/66] loss=0.993028 avg=0.838907 VRAM=38.6GiB | 99.6% done | ETA(epoch): 115s + Train loss: 0.845024 (1269.3s) ETA: 23min + Val loss: 0.853679 [t_0.0-0.2=1.0789 t_0.2-0.4=1.0091 t_0.4-0.6=0.8255 t_0.6-0.8=0.7117 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 23.9/188.5 GiB (12.7%) | VRAM: 38.2/93.1 GiB (41.0%) + +--- Epoch 299/299 (100% done) --- + [Epoch 299][10/66] loss=0.841613 avg=0.801227 VRAM=38.7GiB | 99.7% done | ETA(epoch): 1074s + [Epoch 299][20/66] loss=0.779123 avg=0.821580 VRAM=38.6GiB | 99.8% done | ETA(epoch): 883s + [Epoch 299][30/66] loss=0.932700 avg=0.835999 VRAM=38.7GiB | 99.8% done | ETA(epoch): 692s + [Epoch 299][40/66] loss=0.836977 avg=0.842654 VRAM=38.6GiB | 99.9% done | ETA(epoch): 500s + [Epoch 299][50/66] loss=0.820199 avg=0.840066 VRAM=38.7GiB | 99.9% done | ETA(epoch): 307s + [Epoch 299][60/66] loss=0.838683 avg=0.843451 VRAM=38.6GiB | 100.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 19799 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 19799 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 19799 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 19799 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 19799 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 19799 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 19799 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 19799 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 19799 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 19799 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.845886 (2314.7s) ETA: 0min + Val loss: 0.859610 [t_0.0-0.2=1.0768 t_0.2-0.4=1.0005 t_0.4-0.6=0.8152 t_0.6-0.8=0.7211 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_128_dim_512_H100/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 23.8/188.5 GiB (12.6%) | VRAM: 38.2/93.1 GiB (41.0%) + [LossCurve WARN] could not save loss curve: No module named 'matplotlib' + +====================================================================== +Training complete. +====================================================================== +wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: uploading history steps 2510-2510, summary (0.0s) + m wandb: +wandb: Run summary: +wandb: adapter_lr 1e-05 +wandb: best_val_loss 0.799 +wandb: cpu_pct 0.8 +wandb: epoch 299 +wandb: final_train_loss 0.84589 +wandb: flops/hdc2a_per_sample_gflops 672.0974 +wandb: flops/total_per_sample_gflops 274113.62308 +wandb: flops/transformer_per_sample_gflops 273441.52568 +wandb: global_step 19793 +wandb: gpu_mem_gib 38.20323 +wandb: +10 ... +wandb: +wandb: View run lora_128_dim_512_H100-train-20260428-190804 at: https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100/runs/eo9bzlri +wandb: View project at: https://wandb.ai/hku-xg-boost/lora_128_dim_512_H100 +wandb: Synced 4 W&B file(s), 110 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260428_190804-eo9bzlri/logs diff --git a/training log/trainlog(lora256dim384) b/training log/trainlog(lora256dim384) new file mode 100644 index 0000000000000000000000000000000000000000..55089ab1843a77730684d7eae054413012d1f083 --- /dev/null +++ b/training log/trainlog(lora256dim384) @@ -0,0 +1,3913 @@ + +====================================================================== +Command: train_script.py --name lora_256_dim_384_H200 --batch-size 10 --adapter-lr 1e-3 --lora-lr 1e-4 --adapter-mlp --lora-rank 256 --fusion-dim 384 --seed 42 +====================================================================== +Run dir : output/lora_256_dim_384_H200 +Log file: output/lora_256_dim_384_H200/train.log +GPU: NVIDIA H200 NVL | VRAM: 139.8 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run sa291f8c (0.3s) + m wandb: setting up run sa291f8c (0.3s) + m wandb: setting up run sa291f8c (0.3s) + m wandb: setting up run sa291f8c (0.3s) + m wandb: setting up run sa291f8c (0.3s) + m wandb: setting up run sa291f8c (0.8s) + m wandb: setting up run sa291f8c (0.8s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260424_163729-sa291f8c +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_256_dim_384_H200-train-20260424-163729 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_256_dim_384_H200 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_256_dim_384_H200/runs/sa291f8c + +Run Summary: + name lora_256_dim_384_H200 + fusion MLP (lightweight) + LoRA ON (rank=256) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_256_dim_384_H200 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 384 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 10 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_256_dim_384_H200 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.8/188.4 GiB (6.3%) | VRAM: 0.0/139.8 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H200 NVL (139.8 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.1s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 12.2/188.4 GiB (6.5%) | VRAM: 0.2/139.8 GiB (0.1%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (32.3s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 13.6/188.4 GiB (7.2%) | VRAM: 37.9/139.8 GiB (27.1%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 7.4M params +Control: 0.0M params +Total trainable: 7.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=256, alpha=256.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 78.64M + +Parameter Statistics: + HDC²A Adapter: total=7.4M trainable=7.4M + ControlNet (frozen): total=4212.2M LoRA trainable=78.64M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 7.4M + LoRA 78.64M = 86.02M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~5000 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0442, std=0.5195 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.430925 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.71 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + depth_encoder.conv_stem.6.weight: 0.016602 + semantic_encoder.conv_stem.6.weight: 0.015747 + depth_encoder.conv_stem.3.weight: 0.009644 + semantic_encoder.conv_stem.3.weight: 0.009460 + depth_encoder.proj.weight: 0.009216 + Test result: PASSED +[MEM @ after test] RAM: 14.2/188.4 GiB (7.5%) | VRAM: 38.1/139.8 GiB (27.2%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 604.443 GFLOPs transformer: 273662.180 GFLOPs total: 274266.623 GFLOPs + per-sample forward FLOPs: 274266.623 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=10 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=10 +[Data] Test: 30 samples, batch_size=10 + +====================================================================== +Starting training: 500 epochs × 40 steps = 20000 total steps + batch_size=10, grad_accum=4, world_size=1, effective_bs=40 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 2000, 4000, 6000, 8000, 10000, 11999, 13999, 15999, 17999, 19999] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/499 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/40] loss=0.978456 avg=0.944615 VRAM=39.2GiB | 0.1% done | ETA(epoch): 2905s + [Epoch 0][20/40] loss=0.894454 avg=0.927977 VRAM=39.0GiB | 0.1% done | ETA(epoch): 1199s + [Epoch 0][30/40] loss=0.907944 avg=0.929855 VRAM=39.2GiB | 0.1% done | ETA(epoch): 476s + [Epoch 0][40/40] loss=1.006778 avg=0.924246 VRAM=39.0GiB | 0.2% done | ETA(epoch): 0s + Train loss: 0.924246 (1659.1s) ETA: 13819min + Val loss: 0.986860 [t_0.0-0.2=1.0957 t_0.2-0.4=1.0435 t_0.4-0.6=0.8921 t_0.6-0.8=0.7453 t_0.8-1.0=1.1028] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 1/499 (0% done) --- + [Epoch 1][10/40] loss=0.968902 avg=0.920706 VRAM=39.2GiB | 0.2% done | ETA(epoch): 690s + [Epoch 1][20/40] loss=1.003589 avg=0.926409 VRAM=39.0GiB | 0.3% done | ETA(epoch): 460s + [Epoch 1][30/40] loss=0.891560 avg=0.919979 VRAM=39.2GiB | 0.4% done | ETA(epoch): 230s + [Epoch 1][40/40] loss=0.965952 avg=0.923344 VRAM=39.0GiB | 0.4% done | ETA(epoch): 0s + Train loss: 0.923344 (921.1s) ETA: 10987min + Val loss: 0.917902 [t_0.0-0.2=1.0877 t_0.2-0.4=1.0630 t_0.4-0.6=0.8721 t_0.6-0.8=0.7968 t_0.8-1.0=0.9058] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 2/499 (0% done) --- + [Epoch 2][10/40] loss=0.838640 avg=0.883804 VRAM=39.2GiB | 0.4% done | ETA(epoch): 691s + [Epoch 2][20/40] loss=0.902119 avg=0.908227 VRAM=39.0GiB | 0.5% done | ETA(epoch): 461s + [Epoch 2][30/40] loss=0.843425 avg=0.916535 VRAM=39.2GiB | 0.5% done | ETA(epoch): 230s + [Epoch 2][40/40] loss=1.003684 avg=0.921031 VRAM=39.0GiB | 0.6% done | ETA(epoch): 0s + Train loss: 0.921031 (921.5s) ETA: 10034min + Val loss: 0.967743 [t_0.0-0.2=1.0858 t_0.2-0.4=1.0393 t_0.4-0.6=0.8671 t_0.6-0.8=0.7692 t_0.8-1.0=0.9922] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 3/499 (1% done) --- + [Epoch 3][10/40] loss=0.925177 avg=0.922801 VRAM=39.2GiB | 0.7% done | ETA(epoch): 691s + [Epoch 3][20/40] loss=0.892473 avg=0.924249 VRAM=39.0GiB | 0.7% done | ETA(epoch): 461s + [Epoch 3][30/40] loss=0.870577 avg=0.927510 VRAM=39.2GiB | 0.8% done | ETA(epoch): 230s + [Epoch 3][40/40] loss=0.991791 avg=0.923062 VRAM=39.0GiB | 0.8% done | ETA(epoch): 0s + Train loss: 0.923062 (921.5s) ETA: 9549min + Val loss: 0.981606 [t_0.0-0.2=1.0974 t_0.2-0.4=1.0214 t_0.4-0.6=0.8778 t_0.6-0.8=0.7726 t_0.8-1.0=1.1144] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 4/499 (1% done) --- + [Epoch 4][10/40] loss=0.832469 avg=0.910517 VRAM=39.2GiB | 0.9% done | ETA(epoch): 691s + [Epoch 4][20/40] loss=0.907130 avg=0.911309 VRAM=39.0GiB | 0.9% done | ETA(epoch): 461s + [Epoch 4][30/40] loss=0.837380 avg=0.911010 VRAM=39.2GiB | 0.9% done | ETA(epoch): 230s + [Epoch 4][40/40] loss=0.845103 avg=0.901756 VRAM=39.0GiB | 1.0% done | ETA(epoch): 0s + Train loss: 0.901756 (921.8s) ETA: 9252min + Val loss: 0.965578 [t_0.0-0.2=1.0889 t_0.2-0.4=1.0260 t_0.4-0.6=0.8685 t_0.6-0.8=0.7524 t_0.8-1.0=1.1098] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0004 +[MEM @ epoch 4 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 5/499 (1% done) --- + [Epoch 5][10/40] loss=0.964090 avg=0.912969 VRAM=39.2GiB | 1.1% done | ETA(epoch): 691s + [Epoch 5][20/40] loss=0.910659 avg=0.904111 VRAM=39.0GiB | 1.1% done | ETA(epoch): 461s + [Epoch 5][30/40] loss=0.957333 avg=0.909463 VRAM=39.2GiB | 1.1% done | ETA(epoch): 230s + [Epoch 5][40/40] loss=0.928720 avg=0.904047 VRAM=39.0GiB | 1.2% done | ETA(epoch): 0s + Train loss: 0.904047 (921.3s) ETA: 9048min + Val loss: 0.935062 [t_0.0-0.2=1.0876 t_0.2-0.4=1.0142 t_0.4-0.6=0.8463 t_0.6-0.8=0.7758 t_0.8-1.0=1.0006] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 6/499 (1% done) --- + [Epoch 6][10/40] loss=0.895684 avg=0.895106 VRAM=39.2GiB | 1.2% done | ETA(epoch): 690s + [Epoch 6][20/40] loss=0.860660 avg=0.902362 VRAM=39.0GiB | 1.3% done | ETA(epoch): 460s + [Epoch 6][30/40] loss=0.920628 avg=0.897409 VRAM=39.2GiB | 1.4% done | ETA(epoch): 230s + [Epoch 6][40/40] loss=0.897801 avg=0.892595 VRAM=39.0GiB | 1.4% done | ETA(epoch): 0s + Train loss: 0.892595 (920.6s) ETA: 8897min + Val loss: 0.914257 [t_0.0-0.2=1.0840 t_0.2-0.4=1.0000 t_0.4-0.6=0.8556 t_0.6-0.8=0.7457 t_0.8-1.0=0.8399] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0006 (BEST) + Deleted old checkpoint: checkpoint_epoch_0001 + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 7/499 (1% done) --- + [Epoch 7][10/40] loss=0.944035 avg=0.899419 VRAM=39.2GiB | 1.5% done | ETA(epoch): 690s + [Epoch 7][20/40] loss=0.992395 avg=0.893300 VRAM=39.0GiB | 1.5% done | ETA(epoch): 460s + [Epoch 7][30/40] loss=0.902963 avg=0.901042 VRAM=39.2GiB | 1.6% done | ETA(epoch): 230s + [Epoch 7][40/40] loss=0.907996 avg=0.902340 VRAM=39.0GiB | 1.6% done | ETA(epoch): 0s + Train loss: 0.902340 (920.5s) ETA: 8779min + Val loss: 0.901089 [t_0.0-0.2=1.0961 t_0.2-0.4=0.9792 t_0.4-0.6=0.8659 t_0.6-0.8=0.7639 t_0.8-1.0=0.7777] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0007 (BEST) + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 8/499 (2% done) --- + [Epoch 8][10/40] loss=0.899290 avg=0.889522 VRAM=39.2GiB | 1.7% done | ETA(epoch): 691s + [Epoch 8][20/40] loss=0.878177 avg=0.898925 VRAM=39.0GiB | 1.7% done | ETA(epoch): 461s + [Epoch 8][30/40] loss=0.860018 avg=0.895583 VRAM=39.2GiB | 1.8% done | ETA(epoch): 230s + [Epoch 8][40/40] loss=0.910897 avg=0.889617 VRAM=39.0GiB | 1.8% done | ETA(epoch): 0s + Train loss: 0.889617 (921.2s) ETA: 8685min + Val loss: 0.935518 [t_0.0-0.2=1.0775 t_0.2-0.4=1.0428 t_0.4-0.6=0.8181 t_0.6-0.8=0.7714 t_0.8-1.0=0.9012] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 9/499 (2% done) --- + [Epoch 9][10/40] loss=0.892325 avg=0.898533 VRAM=39.2GiB | 1.8% done | ETA(epoch): 691s + [Epoch 9][20/40] loss=0.895313 avg=0.897207 VRAM=39.0GiB | 1.9% done | ETA(epoch): 461s + [Epoch 9][30/40] loss=0.955273 avg=0.897647 VRAM=39.2GiB | 1.9% done | ETA(epoch): 230s + [Epoch 9][40/40] loss=0.931984 avg=0.900943 VRAM=39.0GiB | 2.0% done | ETA(epoch): 0s + Train loss: 0.900943 (921.7s) ETA: 8606min + Val loss: 0.887891 [t_0.0-0.2=1.0796 t_0.2-0.4=1.0368 t_0.4-0.6=0.8389 t_0.6-0.8=0.7350 t_0.8-1.0=0.8849] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0009 (BEST) + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 10/499 (2% done) --- + [Epoch 10][10/40] loss=0.821933 avg=0.883118 VRAM=39.2GiB | 2.1% done | ETA(epoch): 691s + [Epoch 10][20/40] loss=0.824187 avg=0.884112 VRAM=39.0GiB | 2.1% done | ETA(epoch): 461s + [Epoch 10][30/40] loss=0.918245 avg=0.881949 VRAM=39.2GiB | 2.1% done | ETA(epoch): 231s + [Epoch 10][40/40] loss=0.948225 avg=0.891313 VRAM=39.0GiB | 2.2% done | ETA(epoch): 0s + Train loss: 0.891313 (922.4s) ETA: 8540min + Val loss: 0.885686 [t_0.0-0.2=1.0675 t_0.2-0.4=1.0105 t_0.4-0.6=0.8760 t_0.6-0.8=0.7532 t_0.8-1.0=0.8423] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0010 (BEST) + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 11/499 (2% done) --- + [Epoch 11][10/40] loss=0.845819 avg=0.875477 VRAM=39.2GiB | 2.2% done | ETA(epoch): 692s + [Epoch 11][20/40] loss=0.882443 avg=0.869948 VRAM=39.0GiB | 2.3% done | ETA(epoch): 461s + [Epoch 11][30/40] loss=0.926419 avg=0.880999 VRAM=39.2GiB | 2.4% done | ETA(epoch): 231s + [Epoch 11][40/40] loss=0.926275 avg=0.885754 VRAM=39.0GiB | 2.4% done | ETA(epoch): 0s + Train loss: 0.885754 (922.5s) ETA: 8481min + Val loss: 0.897847 [t_0.0-0.2=1.0843 t_0.2-0.4=0.9671 t_0.4-0.6=0.8154 t_0.6-0.8=0.7808 t_0.8-1.0=0.8364] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 12/499 (2% done) --- + [Epoch 12][10/40] loss=0.900773 avg=0.891739 VRAM=39.2GiB | 2.5% done | ETA(epoch): 691s + [Epoch 12][20/40] loss=0.851485 avg=0.892976 VRAM=39.0GiB | 2.5% done | ETA(epoch): 461s + [Epoch 12][30/40] loss=0.802919 avg=0.889277 VRAM=39.2GiB | 2.5% done | ETA(epoch): 230s + [Epoch 12][40/40] loss=0.925962 avg=0.883696 VRAM=39.0GiB | 2.6% done | ETA(epoch): 0s + Train loss: 0.883696 (921.9s) ETA: 8429min + Val loss: 0.911878 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9904 t_0.4-0.6=0.8997 t_0.6-0.8=0.7535 t_0.8-1.0=0.7735] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 13/499 (3% done) --- + [Epoch 13][10/40] loss=0.998639 avg=0.920901 VRAM=39.2GiB | 2.6% done | ETA(epoch): 691s + [Epoch 13][20/40] loss=0.868953 avg=0.915279 VRAM=39.0GiB | 2.7% done | ETA(epoch): 461s + [Epoch 13][30/40] loss=0.828134 avg=0.904017 VRAM=39.2GiB | 2.8% done | ETA(epoch): 231s + [Epoch 13][40/40] loss=0.864583 avg=0.898942 VRAM=39.0GiB | 2.8% done | ETA(epoch): 0s + Train loss: 0.898942 (922.3s) ETA: 8382min + Val loss: 0.882031 [t_0.0-0.2=1.0728 t_0.2-0.4=0.9862 t_0.4-0.6=0.8875 t_0.6-0.8=0.7458 t_0.8-1.0=0.7999] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0013 (BEST) + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 14/499 (3% done) --- + [Epoch 14][10/40] loss=0.898970 avg=0.899401 VRAM=39.2GiB | 2.9% done | ETA(epoch): 691s + [Epoch 14][20/40] loss=0.825314 avg=0.904444 VRAM=39.0GiB | 2.9% done | ETA(epoch): 461s + [Epoch 14][30/40] loss=0.910102 avg=0.900647 VRAM=39.2GiB | 2.9% done | ETA(epoch): 231s + [Epoch 14][40/40] loss=0.779546 avg=0.894576 VRAM=39.0GiB | 3.0% done | ETA(epoch): 0s + Train loss: 0.894576 (922.1s) ETA: 8339min + Val loss: 0.876412 [t_0.0-0.2=1.0573 t_0.2-0.4=1.0342 t_0.4-0.6=0.8460 t_0.6-0.8=0.7562 t_0.8-1.0=0.7714] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0014 (BEST) + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 15/499 (3% done) --- + [Epoch 15][10/40] loss=0.828090 avg=0.871061 VRAM=39.2GiB | 3.0% done | ETA(epoch): 691s + [Epoch 15][20/40] loss=0.839641 avg=0.882444 VRAM=39.0GiB | 3.1% done | ETA(epoch): 461s + [Epoch 15][30/40] loss=0.947766 avg=0.882162 VRAM=39.2GiB | 3.1% done | ETA(epoch): 230s + [Epoch 15][40/40] loss=0.853161 avg=0.881736 VRAM=39.0GiB | 3.2% done | ETA(epoch): 0s + Train loss: 0.881736 (921.9s) ETA: 8300min + Val loss: 0.917590 [t_0.0-0.2=1.0555 t_0.2-0.4=1.0111 t_0.4-0.6=0.8610 t_0.6-0.8=0.7425 t_0.8-1.0=0.8209] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 16/499 (3% done) --- + [Epoch 16][10/40] loss=0.883140 avg=0.881514 VRAM=39.2GiB | 3.2% done | ETA(epoch): 692s + [Epoch 16][20/40] loss=0.874821 avg=0.877623 VRAM=39.0GiB | 3.3% done | ETA(epoch): 461s + [Epoch 16][30/40] loss=0.832853 avg=0.882983 VRAM=39.2GiB | 3.4% done | ETA(epoch): 230s + [Epoch 16][40/40] loss=0.897467 avg=0.884630 VRAM=39.0GiB | 3.4% done | ETA(epoch): 0s + Train loss: 0.884630 (921.9s) ETA: 8263min + Val loss: 0.900020 [t_0.0-0.2=1.0867 t_0.2-0.4=1.0285 t_0.4-0.6=0.8501 t_0.6-0.8=0.7364 t_0.8-1.0=0.7461] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 17/499 (3% done) --- + [Epoch 17][10/40] loss=0.847486 avg=0.868114 VRAM=39.2GiB | 3.5% done | ETA(epoch): 692s + [Epoch 17][20/40] loss=0.882731 avg=0.871000 VRAM=39.0GiB | 3.5% done | ETA(epoch): 461s + [Epoch 17][30/40] loss=0.888869 avg=0.870954 VRAM=39.2GiB | 3.5% done | ETA(epoch): 231s + [Epoch 17][40/40] loss=0.797519 avg=0.879710 VRAM=39.0GiB | 3.6% done | ETA(epoch): 0s + Train loss: 0.879710 (921.9s) ETA: 8228min + Val loss: 0.879541 [t_0.0-0.2=1.0712 t_0.2-0.4=0.9960 t_0.4-0.6=0.8152 t_0.6-0.8=0.7402 t_0.8-1.0=0.7890] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0017 +[MEM @ epoch 17 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 18/499 (4% done) --- + [Epoch 18][10/40] loss=0.885745 avg=0.891585 VRAM=39.2GiB | 3.6% done | ETA(epoch): 691s + [Epoch 18][20/40] loss=0.834869 avg=0.896071 VRAM=39.0GiB | 3.7% done | ETA(epoch): 461s + [Epoch 18][30/40] loss=0.881383 avg=0.888274 VRAM=39.2GiB | 3.8% done | ETA(epoch): 230s + [Epoch 18][40/40] loss=0.863939 avg=0.883698 VRAM=39.0GiB | 3.8% done | ETA(epoch): 0s + Train loss: 0.883698 (921.7s) ETA: 8195min + Val loss: 0.897656 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9927 t_0.4-0.6=0.8478 t_0.6-0.8=0.7247 t_0.8-1.0=0.8111] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 19/499 (4% done) --- + [Epoch 19][10/40] loss=0.906645 avg=0.882889 VRAM=39.2GiB | 3.9% done | ETA(epoch): 691s + [Epoch 19][20/40] loss=0.937797 avg=0.889365 VRAM=39.0GiB | 3.9% done | ETA(epoch): 461s + [Epoch 19][30/40] loss=0.927451 avg=0.880525 VRAM=39.2GiB | 4.0% done | ETA(epoch): 230s + [Epoch 19][40/40] loss=0.906198 avg=0.880676 VRAM=39.0GiB | 4.0% done | ETA(epoch): 0s + Train loss: 0.880676 (922.0s) ETA: 8164min + Val loss: 0.861397 [t_0.0-0.2=1.0812 t_0.2-0.4=1.0176 t_0.4-0.6=0.8367 t_0.6-0.8=0.7336 t_0.8-1.0=0.7269] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0019 (BEST) + Deleted old checkpoint: checkpoint_epoch_0014 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 20/499 (4% done) --- + [Epoch 20][10/40] loss=0.837344 avg=0.882329 VRAM=39.2GiB | 4.0% done | ETA(epoch): 692s + [Epoch 20][20/40] loss=0.879482 avg=0.865774 VRAM=39.0GiB | 4.1% done | ETA(epoch): 461s + [Epoch 20][30/40] loss=0.870969 avg=0.859290 VRAM=39.2GiB | 4.2% done | ETA(epoch): 231s + [Epoch 20][40/40] loss=0.823260 avg=0.862194 VRAM=39.0GiB | 4.2% done | ETA(epoch): 0s + Train loss: 0.862194 (922.4s) ETA: 8135min + Val loss: 0.883520 [t_0.0-0.2=1.0682 t_0.2-0.4=1.0047 t_0.4-0.6=0.8390 t_0.6-0.8=0.7623 t_0.8-1.0=0.7252] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 21/499 (4% done) --- + [Epoch 21][10/40] loss=0.898443 avg=0.869948 VRAM=39.2GiB | 4.2% done | ETA(epoch): 691s + [Epoch 21][20/40] loss=0.865199 avg=0.879940 VRAM=39.0GiB | 4.3% done | ETA(epoch): 461s + [Epoch 21][30/40] loss=0.865801 avg=0.880178 VRAM=39.2GiB | 4.3% done | ETA(epoch): 231s + [Epoch 21][40/40] loss=1.024682 avg=0.884469 VRAM=39.0GiB | 4.4% done | ETA(epoch): 0s + Train loss: 0.884469 (922.2s) ETA: 8106min + Val loss: 0.876846 [t_0.0-0.2=1.0680 t_0.2-0.4=0.9830 t_0.4-0.6=0.8870 t_0.6-0.8=0.7176 t_0.8-1.0=0.7379] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 22/499 (4% done) --- + [Epoch 22][10/40] loss=0.782796 avg=0.851823 VRAM=39.2GiB | 4.5% done | ETA(epoch): 692s + [Epoch 22][20/40] loss=0.805979 avg=0.844871 VRAM=39.0GiB | 4.5% done | ETA(epoch): 461s + [Epoch 22][30/40] loss=0.850498 avg=0.854691 VRAM=39.2GiB | 4.5% done | ETA(epoch): 231s + [Epoch 22][40/40] loss=0.854467 avg=0.860454 VRAM=39.0GiB | 4.6% done | ETA(epoch): 0s + Train loss: 0.860454 (922.9s) ETA: 8079min + Val loss: 0.866386 [t_0.0-0.2=1.0533 t_0.2-0.4=1.0020 t_0.4-0.6=0.8497 t_0.6-0.8=0.7507 t_0.8-1.0=0.7335] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0022 +[MEM @ epoch 22 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 23/499 (5% done) --- + [Epoch 23][10/40] loss=0.916355 avg=0.863354 VRAM=39.2GiB | 4.7% done | ETA(epoch): 692s + [Epoch 23][20/40] loss=0.876902 avg=0.862354 VRAM=39.0GiB | 4.7% done | ETA(epoch): 461s + [Epoch 23][30/40] loss=0.849537 avg=0.862241 VRAM=39.2GiB | 4.8% done | ETA(epoch): 231s + [Epoch 23][40/40] loss=0.901659 avg=0.870136 VRAM=39.0GiB | 4.8% done | ETA(epoch): 0s + Train loss: 0.870136 (922.8s) ETA: 8053min + Val loss: 0.876556 [t_0.0-0.2=1.0863 t_0.2-0.4=1.0054 t_0.4-0.6=0.8375 t_0.6-0.8=0.7315 t_0.8-1.0=0.7186] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 24/499 (5% done) --- + [Epoch 24][10/40] loss=0.858866 avg=0.868786 VRAM=39.2GiB | 4.9% done | ETA(epoch): 692s + [Epoch 24][20/40] loss=0.857664 avg=0.867764 VRAM=39.0GiB | 4.9% done | ETA(epoch): 461s + [Epoch 24][30/40] loss=0.795641 avg=0.860875 VRAM=39.2GiB | 5.0% done | ETA(epoch): 231s + [Epoch 24][40/40] loss=0.792677 avg=0.862241 VRAM=39.0GiB | 5.0% done | ETA(epoch): 0s + Train loss: 0.862241 (922.7s) ETA: 8027min + Val loss: 0.892556 [t_0.0-0.2=1.0938 t_0.2-0.4=0.9801 t_0.4-0.6=0.8667 t_0.6-0.8=0.7479 t_0.8-1.0=0.7120] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 25/499 (5% done) --- + [Epoch 25][10/40] loss=0.937581 avg=0.882648 VRAM=39.2GiB | 5.1% done | ETA(epoch): 692s + [Epoch 25][20/40] loss=0.865773 avg=0.869613 VRAM=39.0GiB | 5.1% done | ETA(epoch): 461s + [Epoch 25][30/40] loss=0.903117 avg=0.870154 VRAM=39.2GiB | 5.1% done | ETA(epoch): 231s + [Epoch 25][40/40] loss=0.958801 avg=0.872530 VRAM=39.0GiB | 5.2% done | ETA(epoch): 0s + Train loss: 0.872530 (922.3s) ETA: 8002min + Val loss: 0.888530 [t_0.0-0.2=1.0865 t_0.2-0.4=0.9873 t_0.4-0.6=0.8673 t_0.6-0.8=0.7278 t_0.8-1.0=0.7511] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 26/499 (5% done) --- + [Epoch 26][10/40] loss=0.791230 avg=0.866668 VRAM=39.2GiB | 5.2% done | ETA(epoch): 691s + [Epoch 26][20/40] loss=0.859381 avg=0.880564 VRAM=39.0GiB | 5.3% done | ETA(epoch): 461s + [Epoch 26][30/40] loss=0.798637 avg=0.875345 VRAM=39.2GiB | 5.3% done | ETA(epoch): 231s + [Epoch 26][40/40] loss=0.871435 avg=0.875525 VRAM=39.0GiB | 5.4% done | ETA(epoch): 0s + Train loss: 0.875525 (922.2s) ETA: 7978min + Val loss: 0.859988 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9983 t_0.4-0.6=0.8436 t_0.6-0.8=0.7409 t_0.8-1.0=0.7007] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0026 (BEST) + Deleted old checkpoint: checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 27/499 (5% done) --- + [Epoch 27][10/40] loss=0.889111 avg=0.877975 VRAM=39.2GiB | 5.5% done | ETA(epoch): 692s + [Epoch 27][20/40] loss=0.826070 avg=0.880852 VRAM=39.0GiB | 5.5% done | ETA(epoch): 461s + [Epoch 27][30/40] loss=0.830319 avg=0.875812 VRAM=39.2GiB | 5.5% done | ETA(epoch): 231s + [Epoch 27][40/40] loss=0.777936 avg=0.871749 VRAM=39.0GiB | 5.6% done | ETA(epoch): 0s + Train loss: 0.871749 (922.3s) ETA: 7954min + Val loss: 0.897313 [t_0.0-0.2=1.0840 t_0.2-0.4=1.0049 t_0.4-0.6=0.8339 t_0.6-0.8=0.7322 t_0.8-1.0=0.7149] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 28/499 (6% done) --- + [Epoch 28][10/40] loss=0.904138 avg=0.877882 VRAM=39.2GiB | 5.7% done | ETA(epoch): 692s + [Epoch 28][20/40] loss=0.824374 avg=0.873705 VRAM=39.0GiB | 5.7% done | ETA(epoch): 461s + [Epoch 28][30/40] loss=0.882612 avg=0.871123 VRAM=39.2GiB | 5.8% done | ETA(epoch): 231s + [Epoch 28][40/40] loss=0.830171 avg=0.878263 VRAM=39.0GiB | 5.8% done | ETA(epoch): 0s + Train loss: 0.878263 (922.2s) ETA: 7931min + Val loss: 0.872925 [t_0.0-0.2=1.0883 t_0.2-0.4=1.0230 t_0.4-0.6=0.7938 t_0.6-0.8=0.7590 t_0.8-1.0=0.7340] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 29/499 (6% done) --- + [Epoch 29][10/40] loss=0.921005 avg=0.858953 VRAM=39.2GiB | 5.9% done | ETA(epoch): 691s + [Epoch 29][20/40] loss=0.935715 avg=0.890072 VRAM=39.0GiB | 5.9% done | ETA(epoch): 461s + [Epoch 29][30/40] loss=0.876555 avg=0.888869 VRAM=39.2GiB | 5.9% done | ETA(epoch): 231s + [Epoch 29][40/40] loss=0.860823 avg=0.883654 VRAM=39.0GiB | 6.0% done | ETA(epoch): 0s + Train loss: 0.883654 (922.3s) ETA: 7908min + Val loss: 0.872480 [t_0.0-0.2=1.0875 t_0.2-0.4=1.0256 t_0.4-0.6=0.8605 t_0.6-0.8=0.7464 t_0.8-1.0=0.7075] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0029 +[MEM @ epoch 29 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 30/499 (6% done) --- + [Epoch 30][10/40] loss=0.831487 avg=0.854485 VRAM=39.2GiB | 6.0% done | ETA(epoch): 692s + [Epoch 30][20/40] loss=0.861505 avg=0.868452 VRAM=39.0GiB | 6.1% done | ETA(epoch): 461s + [Epoch 30][30/40] loss=0.897540 avg=0.870559 VRAM=39.2GiB | 6.2% done | ETA(epoch): 231s + [Epoch 30][40/40] loss=0.974987 avg=0.869704 VRAM=39.0GiB | 6.2% done | ETA(epoch): 0s + Train loss: 0.869704 (922.6s) ETA: 7886min + Val loss: 0.913632 [t_0.0-0.2=1.0826 t_0.2-0.4=0.9985 t_0.4-0.6=0.8752 t_0.6-0.8=0.7183 t_0.8-1.0=0.7093] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 31/499 (6% done) --- + [Epoch 31][10/40] loss=0.819282 avg=0.867483 VRAM=39.2GiB | 6.2% done | ETA(epoch): 692s + [Epoch 31][20/40] loss=0.912551 avg=0.855515 VRAM=39.0GiB | 6.3% done | ETA(epoch): 461s + [Epoch 31][30/40] loss=0.886656 avg=0.869506 VRAM=39.2GiB | 6.3% done | ETA(epoch): 231s + [Epoch 31][40/40] loss=0.887784 avg=0.875460 VRAM=39.0GiB | 6.4% done | ETA(epoch): 0s + Train loss: 0.875460 (922.8s) ETA: 7864min + Val loss: 0.846410 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9897 t_0.4-0.6=0.8409 t_0.6-0.8=0.7472 t_0.8-1.0=0.7116] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0031 (BEST) + Deleted old checkpoint: checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 32/499 (6% done) --- + [Epoch 32][10/40] loss=0.871734 avg=0.873264 VRAM=39.2GiB | 6.5% done | ETA(epoch): 692s + [Epoch 32][20/40] loss=0.903248 avg=0.874640 VRAM=39.0GiB | 6.5% done | ETA(epoch): 461s + [Epoch 32][30/40] loss=0.942033 avg=0.871655 VRAM=39.2GiB | 6.6% done | ETA(epoch): 231s + [Epoch 32][40/40] loss=0.843181 avg=0.861921 VRAM=39.0GiB | 6.6% done | ETA(epoch): 0s + Train loss: 0.861921 (922.2s) ETA: 7842min + Val loss: 0.900291 [t_0.0-0.2=1.0778 t_0.2-0.4=1.0426 t_0.4-0.6=0.8534 t_0.6-0.8=0.7318 t_0.8-1.0=0.7492] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 33/499 (7% done) --- + [Epoch 33][10/40] loss=0.897104 avg=0.890133 VRAM=39.2GiB | 6.7% done | ETA(epoch): 691s + [Epoch 33][20/40] loss=0.954182 avg=0.880299 VRAM=39.0GiB | 6.7% done | ETA(epoch): 461s + [Epoch 33][30/40] loss=0.819503 avg=0.877732 VRAM=39.2GiB | 6.8% done | ETA(epoch): 230s + [Epoch 33][40/40] loss=0.906930 avg=0.882227 VRAM=39.0GiB | 6.8% done | ETA(epoch): 0s + Train loss: 0.882227 (922.0s) ETA: 7821min + Val loss: 0.873120 [t_0.0-0.2=1.0691 t_0.2-0.4=0.9989 t_0.4-0.6=0.8310 t_0.6-0.8=0.7600 t_0.8-1.0=0.7283] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 34/499 (7% done) --- + [Epoch 34][10/40] loss=0.866546 avg=0.880882 VRAM=39.2GiB | 6.9% done | ETA(epoch): 692s + [Epoch 34][20/40] loss=0.838068 avg=0.869676 VRAM=39.0GiB | 6.9% done | ETA(epoch): 461s + [Epoch 34][30/40] loss=0.908844 avg=0.873856 VRAM=39.2GiB | 7.0% done | ETA(epoch): 231s + [Epoch 34][40/40] loss=0.952879 avg=0.876877 VRAM=39.0GiB | 7.0% done | ETA(epoch): 0s + Train loss: 0.876877 (922.5s) ETA: 7800min + Val loss: 0.865366 [t_0.0-0.2=1.0792 t_0.2-0.4=0.9903 t_0.4-0.6=0.8343 t_0.6-0.8=0.7538 t_0.8-1.0=0.7195] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0034 +[MEM @ epoch 34 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 35/499 (7% done) --- + [Epoch 35][10/40] loss=0.813575 avg=0.851609 VRAM=39.2GiB | 7.0% done | ETA(epoch): 692s + [Epoch 35][20/40] loss=0.888296 avg=0.870251 VRAM=39.0GiB | 7.1% done | ETA(epoch): 461s + [Epoch 35][30/40] loss=0.885688 avg=0.870136 VRAM=39.2GiB | 7.1% done | ETA(epoch): 231s + [Epoch 35][40/40] loss=0.903293 avg=0.869773 VRAM=39.0GiB | 7.2% done | ETA(epoch): 0s + Train loss: 0.869773 (922.9s) ETA: 7779min + Val loss: 0.866335 [t_0.0-0.2=1.0819 t_0.2-0.4=0.9732 t_0.4-0.6=0.8468 t_0.6-0.8=0.7522 t_0.8-1.0=0.7027] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 36/499 (7% done) --- + [Epoch 36][10/40] loss=0.779536 avg=0.839534 VRAM=39.2GiB | 7.2% done | ETA(epoch): 692s + [Epoch 36][20/40] loss=0.904869 avg=0.868391 VRAM=39.0GiB | 7.3% done | ETA(epoch): 462s + [Epoch 36][30/40] loss=0.863293 avg=0.869783 VRAM=39.2GiB | 7.3% done | ETA(epoch): 231s + [Epoch 36][40/40] loss=0.861484 avg=0.863503 VRAM=39.0GiB | 7.4% done | ETA(epoch): 0s + Train loss: 0.863503 (923.4s) ETA: 7759min + Val loss: 0.845747 [t_0.0-0.2=1.0795 t_0.2-0.4=1.0101 t_0.4-0.6=0.8587 t_0.6-0.8=0.7315 t_0.8-1.0=0.7012] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0036 (BEST) + Deleted old checkpoint: checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 37/499 (7% done) --- + [Epoch 37][10/40] loss=0.843777 avg=0.882929 VRAM=39.2GiB | 7.4% done | ETA(epoch): 692s + [Epoch 37][20/40] loss=0.902320 avg=0.869227 VRAM=39.0GiB | 7.5% done | ETA(epoch): 462s + [Epoch 37][30/40] loss=0.865749 avg=0.880664 VRAM=39.2GiB | 7.5% done | ETA(epoch): 231s + [Epoch 37][40/40] loss=0.807502 avg=0.875553 VRAM=39.0GiB | 7.6% done | ETA(epoch): 0s + Train loss: 0.875553 (923.1s) ETA: 7739min + Val loss: 0.892351 [t_0.0-0.2=1.0736 t_0.2-0.4=1.0007 t_0.4-0.6=0.8715 t_0.6-0.8=0.7469 t_0.8-1.0=0.6879] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 38/499 (8% done) --- + [Epoch 38][10/40] loss=0.870498 avg=0.867966 VRAM=39.2GiB | 7.6% done | ETA(epoch): 692s + [Epoch 38][20/40] loss=0.787371 avg=0.873134 VRAM=39.0GiB | 7.7% done | ETA(epoch): 462s + [Epoch 38][30/40] loss=0.882736 avg=0.869616 VRAM=39.2GiB | 7.8% done | ETA(epoch): 231s + [Epoch 38][40/40] loss=0.945294 avg=0.875807 VRAM=39.0GiB | 7.8% done | ETA(epoch): 0s + Train loss: 0.875807 (923.0s) ETA: 7719min + Val loss: 0.882429 [t_0.0-0.2=1.0873 t_0.2-0.4=0.9694 t_0.4-0.6=0.8875 t_0.6-0.8=0.7273 t_0.8-1.0=0.7403] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 39/499 (8% done) --- + [Epoch 39][10/40] loss=0.897767 avg=0.878221 VRAM=39.2GiB | 7.8% done | ETA(epoch): 692s + [Epoch 39][20/40] loss=0.830394 avg=0.867102 VRAM=39.0GiB | 7.9% done | ETA(epoch): 462s + [Epoch 39][30/40] loss=0.929155 avg=0.870270 VRAM=39.2GiB | 8.0% done | ETA(epoch): 231s + [Epoch 39][40/40] loss=0.830110 avg=0.865382 VRAM=39.0GiB | 8.0% done | ETA(epoch): 0s + Train loss: 0.865382 (922.9s) ETA: 7699min + Val loss: 0.900870 [t_0.0-0.2=1.0824 t_0.2-0.4=0.9664 t_0.4-0.6=0.8789 t_0.6-0.8=0.7394 t_0.8-1.0=0.7274] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0039 +[MEM @ epoch 39 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 40/499 (8% done) --- + [Epoch 40][10/40] loss=0.709357 avg=0.855455 VRAM=39.2GiB | 8.1% done | ETA(epoch): 692s + [Epoch 40][20/40] loss=0.851401 avg=0.861787 VRAM=39.0GiB | 8.1% done | ETA(epoch): 461s + [Epoch 40][30/40] loss=0.848610 avg=0.856485 VRAM=39.2GiB | 8.2% done | ETA(epoch): 231s + [Epoch 40][40/40] loss=0.864026 avg=0.862444 VRAM=39.0GiB | 8.2% done | ETA(epoch): 0s + Train loss: 0.862444 (922.8s) ETA: 7679min + Val loss: 0.874779 [t_0.0-0.2=1.0526 t_0.2-0.4=0.9967 t_0.4-0.6=0.8890 t_0.6-0.8=0.7248 t_0.8-1.0=0.7456] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 41/499 (8% done) --- + [Epoch 41][10/40] loss=0.863139 avg=0.873873 VRAM=39.2GiB | 8.2% done | ETA(epoch): 692s + [Epoch 41][20/40] loss=0.902194 avg=0.875459 VRAM=39.0GiB | 8.3% done | ETA(epoch): 461s + [Epoch 41][30/40] loss=0.852054 avg=0.870774 VRAM=39.2GiB | 8.3% done | ETA(epoch): 231s + [Epoch 41][40/40] loss=0.859406 avg=0.868106 VRAM=39.0GiB | 8.4% done | ETA(epoch): 0s + Train loss: 0.868106 (922.8s) ETA: 7659min + Val loss: 0.853553 [t_0.0-0.2=1.0684 t_0.2-0.4=1.0073 t_0.4-0.6=0.8464 t_0.6-0.8=0.7121 t_0.8-1.0=0.7154] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 42/499 (8% done) --- + [Epoch 42][10/40] loss=0.886215 avg=0.837359 VRAM=39.2GiB | 8.5% done | ETA(epoch): 692s + [Epoch 42][20/40] loss=0.782543 avg=0.854740 VRAM=39.0GiB | 8.5% done | ETA(epoch): 461s + [Epoch 42][30/40] loss=0.836883 avg=0.861219 VRAM=39.2GiB | 8.6% done | ETA(epoch): 231s + [Epoch 42][40/40] loss=0.875431 avg=0.864778 VRAM=39.0GiB | 8.6% done | ETA(epoch): 0s + Train loss: 0.864778 (922.6s) ETA: 7640min + Val loss: 0.882114 [t_0.0-0.2=1.0747 t_0.2-0.4=1.0123 t_0.4-0.6=0.8426 t_0.6-0.8=0.7324 t_0.8-1.0=0.7225] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 43/499 (9% done) --- + [Epoch 43][10/40] loss=0.859975 avg=0.861284 VRAM=39.2GiB | 8.6% done | ETA(epoch): 691s + [Epoch 43][20/40] loss=0.735149 avg=0.852463 VRAM=39.0GiB | 8.7% done | ETA(epoch): 461s + [Epoch 43][30/40] loss=0.939865 avg=0.859973 VRAM=39.2GiB | 8.8% done | ETA(epoch): 231s + [Epoch 43][40/40] loss=0.917913 avg=0.863371 VRAM=39.0GiB | 8.8% done | ETA(epoch): 0s + Train loss: 0.863371 (922.3s) ETA: 7620min + Val loss: 0.869057 [t_0.0-0.2=1.0930 t_0.2-0.4=0.9656 t_0.4-0.6=0.8665 t_0.6-0.8=0.7288 t_0.8-1.0=0.7222] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 44/499 (9% done) --- + [Epoch 44][10/40] loss=0.843235 avg=0.840415 VRAM=39.2GiB | 8.8% done | ETA(epoch): 692s + [Epoch 44][20/40] loss=0.870018 avg=0.834280 VRAM=39.0GiB | 8.9% done | ETA(epoch): 461s + [Epoch 44][30/40] loss=0.820751 avg=0.838031 VRAM=39.2GiB | 8.9% done | ETA(epoch): 231s + [Epoch 44][40/40] loss=0.823119 avg=0.850966 VRAM=39.0GiB | 9.0% done | ETA(epoch): 0s + Train loss: 0.850966 (922.6s) ETA: 7601min + Val loss: 0.874749 [t_0.0-0.2=1.0883 t_0.2-0.4=0.9893 t_0.4-0.6=0.8030 t_0.6-0.8=0.7392 t_0.8-1.0=0.7099] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 45/499 (9% done) --- + [Epoch 45][10/40] loss=0.871785 avg=0.860150 VRAM=39.2GiB | 9.0% done | ETA(epoch): 692s + [Epoch 45][20/40] loss=0.954146 avg=0.873659 VRAM=39.0GiB | 9.1% done | ETA(epoch): 461s + [Epoch 45][30/40] loss=0.812194 avg=0.874202 VRAM=39.2GiB | 9.2% done | ETA(epoch): 231s + [Epoch 45][40/40] loss=0.840135 avg=0.875378 VRAM=39.0GiB | 9.2% done | ETA(epoch): 0s + Train loss: 0.875378 (922.5s) ETA: 7582min + Val loss: 0.872954 [t_0.0-0.2=1.0711 t_0.2-0.4=1.0008 t_0.4-0.6=0.8646 t_0.6-0.8=0.7360 t_0.8-1.0=0.7082] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 46/499 (9% done) --- + [Epoch 46][10/40] loss=0.877521 avg=0.908666 VRAM=39.2GiB | 9.2% done | ETA(epoch): 692s + [Epoch 46][20/40] loss=0.912955 avg=0.900053 VRAM=39.0GiB | 9.3% done | ETA(epoch): 462s + [Epoch 46][30/40] loss=0.908650 avg=0.896946 VRAM=39.2GiB | 9.3% done | ETA(epoch): 231s + [Epoch 46][40/40] loss=0.939246 avg=0.889025 VRAM=39.0GiB | 9.4% done | ETA(epoch): 0s + Train loss: 0.889025 (923.1s) ETA: 7563min + Val loss: 0.876850 [t_0.0-0.2=1.0716 t_0.2-0.4=1.0106 t_0.4-0.6=0.8650 t_0.6-0.8=0.7303 t_0.8-1.0=0.7007] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 47/499 (9% done) --- + [Epoch 47][10/40] loss=0.868526 avg=0.858469 VRAM=39.2GiB | 9.4% done | ETA(epoch): 692s + [Epoch 47][20/40] loss=0.857377 avg=0.870529 VRAM=39.0GiB | 9.5% done | ETA(epoch): 461s + [Epoch 47][30/40] loss=0.863340 avg=0.869273 VRAM=39.2GiB | 9.6% done | ETA(epoch): 231s + [Epoch 47][40/40] loss=0.829505 avg=0.871681 VRAM=39.0GiB | 9.6% done | ETA(epoch): 0s + Train loss: 0.871681 (922.3s) ETA: 7544min + Val loss: 0.857923 [t_0.0-0.2=1.0830 t_0.2-0.4=0.9922 t_0.4-0.6=0.8523 t_0.6-0.8=0.7228 t_0.8-1.0=0.7010] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 48/499 (10% done) --- + [Epoch 48][10/40] loss=0.824113 avg=0.845645 VRAM=39.2GiB | 9.7% done | ETA(epoch): 691s + [Epoch 48][20/40] loss=0.878277 avg=0.849481 VRAM=39.0GiB | 9.7% done | ETA(epoch): 461s + [Epoch 48][30/40] loss=0.741491 avg=0.858018 VRAM=39.2GiB | 9.8% done | ETA(epoch): 231s + [Epoch 48][40/40] loss=0.855835 avg=0.860824 VRAM=39.0GiB | 9.8% done | ETA(epoch): 0s + Train loss: 0.860824 (922.4s) ETA: 7525min + Val loss: 0.869927 [t_0.0-0.2=1.0839 t_0.2-0.4=0.9661 t_0.4-0.6=0.8432 t_0.6-0.8=0.7534 t_0.8-1.0=0.6810] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 49/499 (10% done) --- + [Epoch 49][10/40] loss=0.914822 avg=0.861879 VRAM=39.2GiB | 9.8% done | ETA(epoch): 692s + [Epoch 49][20/40] loss=0.850444 avg=0.855472 VRAM=39.0GiB | 9.9% done | ETA(epoch): 461s + [Epoch 49][30/40] loss=0.914112 avg=0.851815 VRAM=39.2GiB | 10.0% done | ETA(epoch): 231s + [Epoch 49][40/40] loss=0.811001 avg=0.851980 VRAM=39.0GiB | 10.0% done | ETA(epoch): 0s + Train loss: 0.851980 (922.5s) ETA: 7507min + Val loss: 0.886233 [t_0.0-0.2=1.0753 t_0.2-0.4=0.9966 t_0.4-0.6=0.8667 t_0.6-0.8=0.7382 t_0.8-1.0=0.6987] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 50/499 (10% done) --- + [MilestoneVis] train_0 step 2000 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 2000 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 2000 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 2000 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 2000 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 2000 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 2000 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 2000 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 2000 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 2000 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 50][10/40] loss=0.946049 avg=0.898737 VRAM=39.2GiB | 10.1% done | ETA(epoch): 2936s + [Epoch 50][20/40] loss=0.853946 avg=0.896694 VRAM=39.0GiB | 10.1% done | ETA(epoch): 1209s + [Epoch 50][30/40] loss=0.890513 avg=0.894235 VRAM=39.2GiB | 10.2% done | ETA(epoch): 480s + [Epoch 50][40/40] loss=0.982501 avg=0.887052 VRAM=39.0GiB | 10.2% done | ETA(epoch): 0s + Train loss: 0.887052 (1670.1s) ETA: 7598min + Val loss: 0.869168 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9985 t_0.4-0.6=0.8584 t_0.6-0.8=0.7288 t_0.8-1.0=0.7051] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 51/499 (10% done) --- + [Epoch 51][10/40] loss=0.990515 avg=0.893308 VRAM=39.2GiB | 10.2% done | ETA(epoch): 692s + [Epoch 51][20/40] loss=0.943918 avg=0.884392 VRAM=39.0GiB | 10.3% done | ETA(epoch): 461s + [Epoch 51][30/40] loss=0.878194 avg=0.876891 VRAM=39.2GiB | 10.3% done | ETA(epoch): 231s + [Epoch 51][40/40] loss=0.776165 avg=0.874485 VRAM=39.0GiB | 10.4% done | ETA(epoch): 0s + Train loss: 0.874485 (922.1s) ETA: 7577min + Val loss: 0.884866 [t_0.0-0.2=1.0773 t_0.2-0.4=0.9943 t_0.4-0.6=0.8827 t_0.6-0.8=0.7337 t_0.8-1.0=0.6855] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 52/499 (10% done) --- + [Epoch 52][10/40] loss=0.877271 avg=0.891632 VRAM=39.2GiB | 10.4% done | ETA(epoch): 691s + [Epoch 52][20/40] loss=0.865721 avg=0.895371 VRAM=39.0GiB | 10.5% done | ETA(epoch): 461s + [Epoch 52][30/40] loss=0.924754 avg=0.882932 VRAM=39.2GiB | 10.5% done | ETA(epoch): 231s + [Epoch 52][40/40] loss=0.872464 avg=0.872350 VRAM=39.0GiB | 10.6% done | ETA(epoch): 0s + Train loss: 0.872350 (922.9s) ETA: 7556min + Val loss: 0.849075 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9707 t_0.4-0.6=0.8576 t_0.6-0.8=0.7290 t_0.8-1.0=0.6855] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 53/499 (11% done) --- + [Epoch 53][10/40] loss=0.782409 avg=0.844408 VRAM=39.2GiB | 10.7% done | ETA(epoch): 692s + [Epoch 53][20/40] loss=0.901273 avg=0.852592 VRAM=39.0GiB | 10.7% done | ETA(epoch): 462s + [Epoch 53][30/40] loss=0.942508 avg=0.861405 VRAM=39.2GiB | 10.8% done | ETA(epoch): 231s + [Epoch 53][40/40] loss=0.877493 avg=0.864363 VRAM=39.0GiB | 10.8% done | ETA(epoch): 0s + Train loss: 0.864363 (923.2s) ETA: 7536min + Val loss: 0.838306 [t_0.0-0.2=1.0733 t_0.2-0.4=1.0037 t_0.4-0.6=0.8159 t_0.6-0.8=0.7288 t_0.8-1.0=0.6837] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0053 (BEST) + Deleted old checkpoint: checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 54/499 (11% done) --- + [Epoch 54][10/40] loss=0.822216 avg=0.857680 VRAM=39.2GiB | 10.8% done | ETA(epoch): 692s + [Epoch 54][20/40] loss=0.784204 avg=0.862788 VRAM=39.0GiB | 10.9% done | ETA(epoch): 461s + [Epoch 54][30/40] loss=0.890880 avg=0.863088 VRAM=39.2GiB | 10.9% done | ETA(epoch): 231s + [Epoch 54][40/40] loss=0.940832 avg=0.873551 VRAM=39.0GiB | 11.0% done | ETA(epoch): 0s + Train loss: 0.873551 (923.0s) ETA: 7516min + Val loss: 0.826004 [t_0.0-0.2=1.0571 t_0.2-0.4=0.9972 t_0.4-0.6=0.8261 t_0.6-0.8=0.7329 t_0.8-1.0=0.6748] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0054 (BEST) + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 55/499 (11% done) --- + [Epoch 55][10/40] loss=0.821787 avg=0.889629 VRAM=39.2GiB | 11.1% done | ETA(epoch): 692s + [Epoch 55][20/40] loss=0.884882 avg=0.864631 VRAM=39.0GiB | 11.1% done | ETA(epoch): 461s + [Epoch 55][30/40] loss=0.893684 avg=0.864646 VRAM=39.2GiB | 11.2% done | ETA(epoch): 231s + [Epoch 55][40/40] loss=0.890203 avg=0.858529 VRAM=39.0GiB | 11.2% done | ETA(epoch): 0s + Train loss: 0.858529 (923.1s) ETA: 7496min + Val loss: 0.876967 [t_0.0-0.2=1.0758 t_0.2-0.4=1.0338 t_0.4-0.6=0.8771 t_0.6-0.8=0.7092 t_0.8-1.0=0.6731] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 56/499 (11% done) --- + [Epoch 56][10/40] loss=0.951691 avg=0.874912 VRAM=39.2GiB | 11.2% done | ETA(epoch): 692s + [Epoch 56][20/40] loss=0.892847 avg=0.867480 VRAM=39.0GiB | 11.3% done | ETA(epoch): 462s + [Epoch 56][30/40] loss=0.803569 avg=0.865798 VRAM=39.2GiB | 11.3% done | ETA(epoch): 231s + [Epoch 56][40/40] loss=0.819910 avg=0.871757 VRAM=39.0GiB | 11.4% done | ETA(epoch): 0s + Train loss: 0.871757 (923.2s) ETA: 7476min + Val loss: 0.894261 [t_0.0-0.2=1.0625 t_0.2-0.4=1.0382 t_0.4-0.6=0.8352 t_0.6-0.8=0.7242 t_0.8-1.0=0.6746] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 57/499 (11% done) --- + [Epoch 57][10/40] loss=0.803137 avg=0.867368 VRAM=39.2GiB | 11.5% done | ETA(epoch): 692s + [Epoch 57][20/40] loss=0.895902 avg=0.849515 VRAM=39.0GiB | 11.5% done | ETA(epoch): 461s + [Epoch 57][30/40] loss=0.891583 avg=0.859392 VRAM=39.2GiB | 11.6% done | ETA(epoch): 231s + [Epoch 57][40/40] loss=0.733878 avg=0.856194 VRAM=39.0GiB | 11.6% done | ETA(epoch): 0s + Train loss: 0.856194 (922.9s) ETA: 7456min + Val loss: 0.863930 [t_0.0-0.2=1.0646 t_0.2-0.4=1.0360 t_0.4-0.6=0.8580 t_0.6-0.8=0.7466 t_0.8-1.0=0.6784] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0057 +[MEM @ epoch 57 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 58/499 (12% done) --- + [Epoch 58][10/40] loss=0.833355 avg=0.843152 VRAM=39.2GiB | 11.7% done | ETA(epoch): 693s + [Epoch 58][20/40] loss=0.959177 avg=0.859249 VRAM=39.0GiB | 11.7% done | ETA(epoch): 462s + [Epoch 58][30/40] loss=0.743541 avg=0.863265 VRAM=39.2GiB | 11.8% done | ETA(epoch): 231s + [Epoch 58][40/40] loss=0.781486 avg=0.868873 VRAM=39.0GiB | 11.8% done | ETA(epoch): 0s + Train loss: 0.868873 (923.4s) ETA: 7436min + Val loss: 0.869512 [t_0.0-0.2=1.0660 t_0.2-0.4=1.0089 t_0.4-0.6=0.7982 t_0.6-0.8=0.7160 t_0.8-1.0=0.6754] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 59/499 (12% done) --- + [Epoch 59][10/40] loss=0.949812 avg=0.874021 VRAM=39.2GiB | 11.8% done | ETA(epoch): 692s + [Epoch 59][20/40] loss=0.733623 avg=0.873744 VRAM=39.0GiB | 11.9% done | ETA(epoch): 462s + [Epoch 59][30/40] loss=0.878021 avg=0.875715 VRAM=39.2GiB | 11.9% done | ETA(epoch): 231s + [Epoch 59][40/40] loss=0.824253 avg=0.867757 VRAM=39.0GiB | 12.0% done | ETA(epoch): 0s + Train loss: 0.867757 (923.6s) ETA: 7416min + Val loss: 0.869073 [t_0.0-0.2=1.0876 t_0.2-0.4=0.9538 t_0.4-0.6=0.8663 t_0.6-0.8=0.7237 t_0.8-1.0=0.6642] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 60/499 (12% done) --- + [Epoch 60][10/40] loss=0.909290 avg=0.866957 VRAM=39.2GiB | 12.0% done | ETA(epoch): 693s + [Epoch 60][20/40] loss=0.967161 avg=0.875922 VRAM=39.0GiB | 12.1% done | ETA(epoch): 462s + [Epoch 60][30/40] loss=0.804985 avg=0.871724 VRAM=39.2GiB | 12.2% done | ETA(epoch): 231s + [Epoch 60][40/40] loss=0.857048 avg=0.871919 VRAM=39.0GiB | 12.2% done | ETA(epoch): 0s + Train loss: 0.871919 (923.7s) ETA: 7397min + Val loss: 0.878089 [t_0.0-0.2=1.0866 t_0.2-0.4=0.9701 t_0.4-0.6=0.8605 t_0.6-0.8=0.7209 t_0.8-1.0=0.6845] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 61/499 (12% done) --- + [Epoch 61][10/40] loss=0.862619 avg=0.895340 VRAM=39.2GiB | 12.2% done | ETA(epoch): 693s + [Epoch 61][20/40] loss=0.847083 avg=0.882044 VRAM=39.0GiB | 12.3% done | ETA(epoch): 462s + [Epoch 61][30/40] loss=0.825991 avg=0.875672 VRAM=39.2GiB | 12.3% done | ETA(epoch): 231s + [Epoch 61][40/40] loss=0.785760 avg=0.863804 VRAM=39.0GiB | 12.4% done | ETA(epoch): 0s + Train loss: 0.863804 (923.6s) ETA: 7378min + Val loss: 0.869453 [t_0.0-0.2=1.0823 t_0.2-0.4=0.9999 t_0.4-0.6=0.8440 t_0.6-0.8=0.7212 t_0.8-1.0=0.6695] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 62/499 (12% done) --- + [Epoch 62][10/40] loss=0.835077 avg=0.861686 VRAM=39.2GiB | 12.4% done | ETA(epoch): 693s + [Epoch 62][20/40] loss=0.876879 avg=0.881088 VRAM=39.0GiB | 12.5% done | ETA(epoch): 462s + [Epoch 62][30/40] loss=0.877056 avg=0.874052 VRAM=39.2GiB | 12.6% done | ETA(epoch): 231s + [Epoch 62][40/40] loss=0.919596 avg=0.871172 VRAM=39.0GiB | 12.6% done | ETA(epoch): 0s + Train loss: 0.871172 (923.5s) ETA: 7358min + Val loss: 0.884262 [t_0.0-0.2=1.0693 t_0.2-0.4=1.0353 t_0.4-0.6=0.8796 t_0.6-0.8=0.7145 t_0.8-1.0=0.6788] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 63/499 (13% done) --- + [Epoch 63][10/40] loss=0.834151 avg=0.841719 VRAM=39.2GiB | 12.7% done | ETA(epoch): 692s + [Epoch 63][20/40] loss=0.847129 avg=0.851657 VRAM=39.0GiB | 12.7% done | ETA(epoch): 462s + [Epoch 63][30/40] loss=0.864792 avg=0.866109 VRAM=39.2GiB | 12.8% done | ETA(epoch): 231s + [Epoch 63][40/40] loss=0.922253 avg=0.873471 VRAM=39.0GiB | 12.8% done | ETA(epoch): 0s + Train loss: 0.873471 (923.3s) ETA: 7339min + Val loss: 0.873942 [t_0.0-0.2=1.0614 t_0.2-0.4=0.9994 t_0.4-0.6=0.8747 t_0.6-0.8=0.7324 t_0.8-1.0=0.6739] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 64/499 (13% done) --- + [Epoch 64][10/40] loss=0.762531 avg=0.868244 VRAM=39.2GiB | 12.8% done | ETA(epoch): 693s + [Epoch 64][20/40] loss=0.857733 avg=0.854216 VRAM=39.0GiB | 12.9% done | ETA(epoch): 462s + [Epoch 64][30/40] loss=0.935990 avg=0.849511 VRAM=39.2GiB | 13.0% done | ETA(epoch): 231s + [Epoch 64][40/40] loss=0.880749 avg=0.851054 VRAM=39.0GiB | 13.0% done | ETA(epoch): 0s + Train loss: 0.851054 (924.2s) ETA: 7320min + Val loss: 0.864872 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9969 t_0.4-0.6=0.8336 t_0.6-0.8=0.7166 t_0.8-1.0=0.6856] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 65/499 (13% done) --- + [Epoch 65][10/40] loss=0.769771 avg=0.833476 VRAM=39.2GiB | 13.1% done | ETA(epoch): 693s + [Epoch 65][20/40] loss=0.881534 avg=0.841024 VRAM=39.0GiB | 13.1% done | ETA(epoch): 462s + [Epoch 65][30/40] loss=0.750270 avg=0.850781 VRAM=39.2GiB | 13.2% done | ETA(epoch): 231s + [Epoch 65][40/40] loss=0.913279 avg=0.855572 VRAM=39.0GiB | 13.2% done | ETA(epoch): 0s + Train loss: 0.855572 (924.1s) ETA: 7301min + Val loss: 0.844522 [t_0.0-0.2=1.0710 t_0.2-0.4=1.0040 t_0.4-0.6=0.8542 t_0.6-0.8=0.6997 t_0.8-1.0=0.6699] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 66/499 (13% done) --- + [Epoch 66][10/40] loss=0.821102 avg=0.871927 VRAM=39.2GiB | 13.2% done | ETA(epoch): 693s + [Epoch 66][20/40] loss=0.932351 avg=0.868164 VRAM=39.0GiB | 13.3% done | ETA(epoch): 462s + [Epoch 66][30/40] loss=0.912781 avg=0.875045 VRAM=39.2GiB | 13.4% done | ETA(epoch): 231s + [Epoch 66][40/40] loss=0.893882 avg=0.872329 VRAM=39.0GiB | 13.4% done | ETA(epoch): 0s + Train loss: 0.872329 (923.8s) ETA: 7282min + Val loss: 0.865894 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0278 t_0.4-0.6=0.8265 t_0.6-0.8=0.7570 t_0.8-1.0=0.6764] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 67/499 (13% done) --- + [Epoch 67][10/40] loss=0.748731 avg=0.853190 VRAM=39.2GiB | 13.5% done | ETA(epoch): 693s + [Epoch 67][20/40] loss=0.901349 avg=0.864930 VRAM=39.0GiB | 13.5% done | ETA(epoch): 462s + [Epoch 67][30/40] loss=0.865050 avg=0.862026 VRAM=39.2GiB | 13.6% done | ETA(epoch): 231s + [Epoch 67][40/40] loss=0.853240 avg=0.863207 VRAM=39.0GiB | 13.6% done | ETA(epoch): 0s + Train loss: 0.863207 (923.8s) ETA: 7263min + Val loss: 0.881544 [t_0.0-0.2=1.0830 t_0.2-0.4=0.9814 t_0.4-0.6=0.8776 t_0.6-0.8=0.7192 t_0.8-1.0=0.6650] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 68/499 (14% done) --- + [Epoch 68][10/40] loss=0.819957 avg=0.836627 VRAM=39.2GiB | 13.7% done | ETA(epoch): 693s + [Epoch 68][20/40] loss=0.757370 avg=0.850679 VRAM=39.0GiB | 13.7% done | ETA(epoch): 462s + [Epoch 68][30/40] loss=0.898454 avg=0.851295 VRAM=39.2GiB | 13.8% done | ETA(epoch): 231s + [Epoch 68][40/40] loss=0.886388 avg=0.848742 VRAM=39.0GiB | 13.8% done | ETA(epoch): 0s + Train loss: 0.848742 (924.0s) ETA: 7244min + Val loss: 0.879476 [t_0.0-0.2=1.0549 t_0.2-0.4=0.9743 t_0.4-0.6=0.8770 t_0.6-0.8=0.7342 t_0.8-1.0=0.6591] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 69/499 (14% done) --- + [Epoch 69][10/40] loss=0.930194 avg=0.873417 VRAM=39.2GiB | 13.9% done | ETA(epoch): 693s + [Epoch 69][20/40] loss=0.811526 avg=0.875635 VRAM=39.0GiB | 13.9% done | ETA(epoch): 462s + [Epoch 69][30/40] loss=0.993024 avg=0.878410 VRAM=39.2GiB | 14.0% done | ETA(epoch): 231s + [Epoch 69][40/40] loss=0.746176 avg=0.876026 VRAM=39.0GiB | 14.0% done | ETA(epoch): 0s + Train loss: 0.876026 (923.8s) ETA: 7226min + Val loss: 0.835539 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9960 t_0.4-0.6=0.8483 t_0.6-0.8=0.7377 t_0.8-1.0=0.6562] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 70/499 (14% done) --- + [Epoch 70][10/40] loss=0.927519 avg=0.862903 VRAM=39.2GiB | 14.1% done | ETA(epoch): 693s + [Epoch 70][20/40] loss=0.958190 avg=0.880338 VRAM=39.0GiB | 14.1% done | ETA(epoch): 462s + [Epoch 70][30/40] loss=0.825169 avg=0.868872 VRAM=39.2GiB | 14.1% done | ETA(epoch): 231s + [Epoch 70][40/40] loss=0.919512 avg=0.864020 VRAM=39.0GiB | 14.2% done | ETA(epoch): 0s + Train loss: 0.864020 (924.4s) ETA: 7207min + Val loss: 0.875530 [t_0.0-0.2=1.0663 t_0.2-0.4=1.0389 t_0.4-0.6=0.8600 t_0.6-0.8=0.7198 t_0.8-1.0=0.6731] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 71/499 (14% done) --- + [Epoch 71][10/40] loss=0.948029 avg=0.875604 VRAM=39.2GiB | 14.2% done | ETA(epoch): 693s + [Epoch 71][20/40] loss=0.836969 avg=0.867921 VRAM=39.0GiB | 14.3% done | ETA(epoch): 462s + [Epoch 71][30/40] loss=0.864805 avg=0.858277 VRAM=39.2GiB | 14.3% done | ETA(epoch): 231s + [Epoch 71][40/40] loss=0.972766 avg=0.862946 VRAM=39.0GiB | 14.4% done | ETA(epoch): 0s + Train loss: 0.862946 (924.5s) ETA: 7188min + Val loss: 0.888013 [t_0.0-0.2=1.0650 t_0.2-0.4=1.0063 t_0.4-0.6=0.8704 t_0.6-0.8=0.7188 t_0.8-1.0=0.6792] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 72/499 (14% done) --- + [Epoch 72][10/40] loss=0.866174 avg=0.891387 VRAM=39.2GiB | 14.4% done | ETA(epoch): 693s + [Epoch 72][20/40] loss=0.947650 avg=0.880293 VRAM=39.0GiB | 14.5% done | ETA(epoch): 462s + [Epoch 72][30/40] loss=0.820226 avg=0.876373 VRAM=39.2GiB | 14.5% done | ETA(epoch): 231s + [Epoch 72][40/40] loss=0.804941 avg=0.869592 VRAM=39.0GiB | 14.6% done | ETA(epoch): 0s + Train loss: 0.869592 (924.3s) ETA: 7170min + Val loss: 0.847730 [t_0.0-0.2=1.0597 t_0.2-0.4=1.0188 t_0.4-0.6=0.8293 t_0.6-0.8=0.7287 t_0.8-1.0=0.6571] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 73/499 (15% done) --- + [Epoch 73][10/40] loss=0.903603 avg=0.883783 VRAM=39.2GiB | 14.6% done | ETA(epoch): 693s + [Epoch 73][20/40] loss=0.902314 avg=0.865934 VRAM=39.0GiB | 14.7% done | ETA(epoch): 462s + [Epoch 73][30/40] loss=0.847107 avg=0.878755 VRAM=39.2GiB | 14.8% done | ETA(epoch): 231s + [Epoch 73][40/40] loss=0.948428 avg=0.869744 VRAM=39.0GiB | 14.8% done | ETA(epoch): 0s + Train loss: 0.869744 (923.8s) ETA: 7151min + Val loss: 0.836778 [t_0.0-0.2=1.0558 t_0.2-0.4=0.9594 t_0.4-0.6=0.8246 t_0.6-0.8=0.7486 t_0.8-1.0=0.6792] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 74/499 (15% done) --- + [Epoch 74][10/40] loss=0.815951 avg=0.860040 VRAM=39.2GiB | 14.8% done | ETA(epoch): 693s + [Epoch 74][20/40] loss=0.852658 avg=0.860013 VRAM=39.0GiB | 14.9% done | ETA(epoch): 462s + [Epoch 74][30/40] loss=0.827801 avg=0.858247 VRAM=39.2GiB | 14.9% done | ETA(epoch): 231s + [Epoch 74][40/40] loss=0.847359 avg=0.858299 VRAM=39.0GiB | 15.0% done | ETA(epoch): 0s + Train loss: 0.858299 (923.7s) ETA: 7133min + Val loss: 0.863476 [t_0.0-0.2=1.0806 t_0.2-0.4=0.9637 t_0.4-0.6=0.8701 t_0.6-0.8=0.7289 t_0.8-1.0=0.6648] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 75/499 (15% done) --- + [Epoch 75][10/40] loss=0.839905 avg=0.846725 VRAM=39.2GiB | 15.0% done | ETA(epoch): 692s + [Epoch 75][20/40] loss=0.789883 avg=0.856471 VRAM=39.0GiB | 15.1% done | ETA(epoch): 461s + [Epoch 75][30/40] loss=0.929604 avg=0.858390 VRAM=39.2GiB | 15.2% done | ETA(epoch): 231s + [Epoch 75][40/40] loss=0.850902 avg=0.861820 VRAM=39.0GiB | 15.2% done | ETA(epoch): 0s + Train loss: 0.861820 (923.6s) ETA: 7115min + Val loss: 0.847219 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9692 t_0.4-0.6=0.8407 t_0.6-0.8=0.7218 t_0.8-1.0=0.6619] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 76/499 (15% done) --- + [Epoch 76][10/40] loss=0.837798 avg=0.852340 VRAM=39.2GiB | 15.2% done | ETA(epoch): 693s + [Epoch 76][20/40] loss=0.914599 avg=0.863462 VRAM=39.0GiB | 15.3% done | ETA(epoch): 462s + [Epoch 76][30/40] loss=0.825622 avg=0.864165 VRAM=39.2GiB | 15.3% done | ETA(epoch): 231s + [Epoch 76][40/40] loss=0.850889 avg=0.859286 VRAM=39.0GiB | 15.4% done | ETA(epoch): 0s + Train loss: 0.859286 (924.3s) ETA: 7096min + Val loss: 0.836799 [t_0.0-0.2=1.0659 t_0.2-0.4=0.9808 t_0.4-0.6=0.8581 t_0.6-0.8=0.7244 t_0.8-1.0=0.6799] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 77/499 (15% done) --- + [Epoch 77][10/40] loss=0.961338 avg=0.859063 VRAM=39.2GiB | 15.4% done | ETA(epoch): 694s + [Epoch 77][20/40] loss=0.794480 avg=0.849545 VRAM=39.0GiB | 15.5% done | ETA(epoch): 462s + [Epoch 77][30/40] loss=0.855846 avg=0.850552 VRAM=39.2GiB | 15.6% done | ETA(epoch): 231s + [Epoch 77][40/40] loss=0.829864 avg=0.856846 VRAM=39.0GiB | 15.6% done | ETA(epoch): 0s + Train loss: 0.856846 (924.9s) ETA: 7078min + Val loss: 0.881741 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0007 t_0.4-0.6=0.8492 t_0.6-0.8=0.7244 t_0.8-1.0=0.6864] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 78/499 (16% done) --- + [Epoch 78][10/40] loss=0.855925 avg=0.894584 VRAM=39.2GiB | 15.7% done | ETA(epoch): 694s + [Epoch 78][20/40] loss=0.794564 avg=0.876338 VRAM=39.0GiB | 15.7% done | ETA(epoch): 462s + [Epoch 78][30/40] loss=0.975719 avg=0.870354 VRAM=39.2GiB | 15.8% done | ETA(epoch): 231s + [Epoch 78][40/40] loss=0.908708 avg=0.867222 VRAM=39.0GiB | 15.8% done | ETA(epoch): 0s + Train loss: 0.867222 (924.6s) ETA: 7060min + Val loss: 0.854210 [t_0.0-0.2=1.0668 t_0.2-0.4=1.0434 t_0.4-0.6=0.8386 t_0.6-0.8=0.7058 t_0.8-1.0=0.6672] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 79/499 (16% done) --- + [Epoch 79][10/40] loss=0.845906 avg=0.839486 VRAM=39.2GiB | 15.8% done | ETA(epoch): 693s + [Epoch 79][20/40] loss=0.891869 avg=0.843905 VRAM=39.0GiB | 15.9% done | ETA(epoch): 462s + [Epoch 79][30/40] loss=0.903025 avg=0.852230 VRAM=39.2GiB | 16.0% done | ETA(epoch): 231s + [Epoch 79][40/40] loss=0.843299 avg=0.856096 VRAM=39.0GiB | 16.0% done | ETA(epoch): 0s + Train loss: 0.856096 (924.2s) ETA: 7042min + Val loss: 0.858322 [t_0.0-0.2=1.0752 t_0.2-0.4=0.9996 t_0.4-0.6=0.8607 t_0.6-0.8=0.7157 t_0.8-1.0=0.6819] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 80/499 (16% done) --- + [Epoch 80][10/40] loss=0.941461 avg=0.875762 VRAM=39.2GiB | 16.1% done | ETA(epoch): 693s + [Epoch 80][20/40] loss=0.862085 avg=0.857181 VRAM=39.0GiB | 16.1% done | ETA(epoch): 462s + [Epoch 80][30/40] loss=0.847827 avg=0.860436 VRAM=39.2GiB | 16.2% done | ETA(epoch): 231s + [Epoch 80][40/40] loss=0.850011 avg=0.859935 VRAM=39.0GiB | 16.2% done | ETA(epoch): 0s + Train loss: 0.859935 (924.4s) ETA: 7024min + Val loss: 0.867787 [t_0.0-0.2=1.0621 t_0.2-0.4=0.9950 t_0.4-0.6=0.8483 t_0.6-0.8=0.7370 t_0.8-1.0=0.6586] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 81/499 (16% done) --- + [Epoch 81][10/40] loss=0.885796 avg=0.865717 VRAM=39.2GiB | 16.2% done | ETA(epoch): 693s + [Epoch 81][20/40] loss=0.810482 avg=0.873283 VRAM=39.0GiB | 16.3% done | ETA(epoch): 462s + [Epoch 81][30/40] loss=0.946136 avg=0.874064 VRAM=39.2GiB | 16.4% done | ETA(epoch): 231s + [Epoch 81][40/40] loss=0.774998 avg=0.874383 VRAM=39.0GiB | 16.4% done | ETA(epoch): 0s + Train loss: 0.874383 (924.1s) ETA: 7006min + Val loss: 0.852067 [t_0.0-0.2=1.0775 t_0.2-0.4=1.0067 t_0.4-0.6=0.8554 t_0.6-0.8=0.7399 t_0.8-1.0=0.6715] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 82/499 (16% done) --- + [Epoch 82][10/40] loss=0.869185 avg=0.850931 VRAM=39.2GiB | 16.4% done | ETA(epoch): 693s + [Epoch 82][20/40] loss=0.870369 avg=0.864935 VRAM=39.0GiB | 16.5% done | ETA(epoch): 462s + [Epoch 82][30/40] loss=0.889074 avg=0.861498 VRAM=39.2GiB | 16.6% done | ETA(epoch): 231s + [Epoch 82][40/40] loss=0.764327 avg=0.865826 VRAM=39.0GiB | 16.6% done | ETA(epoch): 0s + Train loss: 0.865826 (924.3s) ETA: 6987min + Val loss: 0.873937 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9623 t_0.4-0.6=0.8369 t_0.6-0.8=0.7267 t_0.8-1.0=0.6638] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 83/499 (17% done) --- + [Epoch 83][10/40] loss=0.897902 avg=0.869421 VRAM=39.2GiB | 16.7% done | ETA(epoch): 693s + [Epoch 83][20/40] loss=0.925874 avg=0.868235 VRAM=39.0GiB | 16.7% done | ETA(epoch): 462s + [Epoch 83][30/40] loss=0.795947 avg=0.861613 VRAM=39.2GiB | 16.8% done | ETA(epoch): 231s + [Epoch 83][40/40] loss=0.915204 avg=0.861135 VRAM=39.0GiB | 16.8% done | ETA(epoch): 0s + Train loss: 0.861135 (924.5s) ETA: 6969min + Val loss: 0.863411 [t_0.0-0.2=1.0806 t_0.2-0.4=1.0019 t_0.4-0.6=0.8383 t_0.6-0.8=0.7165 t_0.8-1.0=0.6750] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 84/499 (17% done) --- + [Epoch 84][10/40] loss=0.889919 avg=0.858672 VRAM=39.2GiB | 16.9% done | ETA(epoch): 693s + [Epoch 84][20/40] loss=0.918402 avg=0.868685 VRAM=39.0GiB | 16.9% done | ETA(epoch): 462s + [Epoch 84][30/40] loss=0.869929 avg=0.877125 VRAM=39.2GiB | 17.0% done | ETA(epoch): 231s + [Epoch 84][40/40] loss=0.805702 avg=0.869646 VRAM=39.0GiB | 17.0% done | ETA(epoch): 0s + Train loss: 0.869646 (924.4s) ETA: 6952min + Val loss: 0.824518 [t_0.0-0.2=1.0646 t_0.2-0.4=0.9719 t_0.4-0.6=0.8324 t_0.6-0.8=0.7419 t_0.8-1.0=0.6678] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0084 (BEST) + Deleted old checkpoint: checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 85/499 (17% done) --- + [Epoch 85][10/40] loss=0.782905 avg=0.844813 VRAM=39.2GiB | 17.1% done | ETA(epoch): 693s + [Epoch 85][20/40] loss=0.825057 avg=0.840640 VRAM=39.0GiB | 17.1% done | ETA(epoch): 462s + [Epoch 85][30/40] loss=0.859323 avg=0.843578 VRAM=39.2GiB | 17.2% done | ETA(epoch): 231s + [Epoch 85][40/40] loss=0.887350 avg=0.850601 VRAM=39.0GiB | 17.2% done | ETA(epoch): 0s + Train loss: 0.850601 (924.4s) ETA: 6934min + Val loss: 0.825909 [t_0.0-0.2=1.0548 t_0.2-0.4=1.0042 t_0.4-0.6=0.8318 t_0.6-0.8=0.7299 t_0.8-1.0=0.6644] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 86/499 (17% done) --- + [Epoch 86][10/40] loss=0.900629 avg=0.838394 VRAM=39.2GiB | 17.2% done | ETA(epoch): 693s + [Epoch 86][20/40] loss=0.858190 avg=0.837546 VRAM=39.0GiB | 17.3% done | ETA(epoch): 462s + [Epoch 86][30/40] loss=0.781943 avg=0.841791 VRAM=39.2GiB | 17.3% done | ETA(epoch): 231s + [Epoch 86][40/40] loss=0.856543 avg=0.841517 VRAM=39.0GiB | 17.4% done | ETA(epoch): 0s + Train loss: 0.841517 (924.6s) ETA: 6916min + Val loss: 0.872269 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9916 t_0.4-0.6=0.8374 t_0.6-0.8=0.7208 t_0.8-1.0=0.6838] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 87/499 (17% done) --- + [Epoch 87][10/40] loss=0.831332 avg=0.893038 VRAM=39.2GiB | 17.4% done | ETA(epoch): 693s + [Epoch 87][20/40] loss=0.776498 avg=0.862707 VRAM=39.0GiB | 17.5% done | ETA(epoch): 462s + [Epoch 87][30/40] loss=0.891698 avg=0.867524 VRAM=39.2GiB | 17.5% done | ETA(epoch): 231s + [Epoch 87][40/40] loss=0.834328 avg=0.868305 VRAM=39.0GiB | 17.6% done | ETA(epoch): 0s + Train loss: 0.868305 (924.4s) ETA: 6898min + Val loss: 0.875932 [t_0.0-0.2=1.0652 t_0.2-0.4=0.9976 t_0.4-0.6=0.8431 t_0.6-0.8=0.7288 t_0.8-1.0=0.6919] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0087 +[MEM @ epoch 87 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 88/499 (18% done) --- + [Epoch 88][10/40] loss=0.911970 avg=0.858130 VRAM=39.2GiB | 17.6% done | ETA(epoch): 693s + [Epoch 88][20/40] loss=0.823961 avg=0.866708 VRAM=39.0GiB | 17.7% done | ETA(epoch): 462s + [Epoch 88][30/40] loss=0.794398 avg=0.865187 VRAM=39.2GiB | 17.8% done | ETA(epoch): 231s + [Epoch 88][40/40] loss=0.884031 avg=0.858494 VRAM=39.0GiB | 17.8% done | ETA(epoch): 0s + Train loss: 0.858494 (924.4s) ETA: 6880min + Val loss: 0.863721 [t_0.0-0.2=1.0883 t_0.2-0.4=1.0028 t_0.4-0.6=0.8352 t_0.6-0.8=0.7283 t_0.8-1.0=0.6709] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 89/499 (18% done) --- + [Epoch 89][10/40] loss=0.858440 avg=0.865142 VRAM=39.2GiB | 17.8% done | ETA(epoch): 693s + [Epoch 89][20/40] loss=0.770328 avg=0.858356 VRAM=39.0GiB | 17.9% done | ETA(epoch): 462s + [Epoch 89][30/40] loss=0.824228 avg=0.861111 VRAM=39.2GiB | 17.9% done | ETA(epoch): 231s + [Epoch 89][40/40] loss=0.839623 avg=0.870711 VRAM=39.0GiB | 18.0% done | ETA(epoch): 0s + Train loss: 0.870711 (924.0s) ETA: 6862min + Val loss: 0.863005 [t_0.0-0.2=1.0893 t_0.2-0.4=0.9825 t_0.4-0.6=0.9109 t_0.6-0.8=0.7075 t_0.8-1.0=0.6761] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 90/499 (18% done) --- + [Epoch 90][10/40] loss=0.852845 avg=0.848450 VRAM=39.2GiB | 18.1% done | ETA(epoch): 693s + [Epoch 90][20/40] loss=0.889139 avg=0.867826 VRAM=39.0GiB | 18.1% done | ETA(epoch): 462s + [Epoch 90][30/40] loss=0.863346 avg=0.860647 VRAM=39.2GiB | 18.1% done | ETA(epoch): 231s + [Epoch 90][40/40] loss=0.883968 avg=0.863995 VRAM=39.0GiB | 18.2% done | ETA(epoch): 0s + Train loss: 0.863995 (924.0s) ETA: 6844min + Val loss: 0.843476 [t_0.0-0.2=1.0733 t_0.2-0.4=1.0059 t_0.4-0.6=0.8605 t_0.6-0.8=0.7099 t_0.8-1.0=0.6607] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 91/499 (18% done) --- + [Epoch 91][10/40] loss=1.002630 avg=0.864852 VRAM=39.2GiB | 18.2% done | ETA(epoch): 693s + [Epoch 91][20/40] loss=0.824472 avg=0.844356 VRAM=39.0GiB | 18.3% done | ETA(epoch): 462s + [Epoch 91][30/40] loss=0.938066 avg=0.848787 VRAM=39.2GiB | 18.4% done | ETA(epoch): 231s + [Epoch 91][40/40] loss=0.937951 avg=0.857975 VRAM=39.0GiB | 18.4% done | ETA(epoch): 0s + Train loss: 0.857975 (924.0s) ETA: 6827min + Val loss: 0.861976 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9609 t_0.4-0.6=0.8301 t_0.6-0.8=0.7466 t_0.8-1.0=0.6727] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 92/499 (18% done) --- + [Epoch 92][10/40] loss=0.886625 avg=0.883842 VRAM=39.2GiB | 18.4% done | ETA(epoch): 693s + [Epoch 92][20/40] loss=0.908648 avg=0.891669 VRAM=39.0GiB | 18.5% done | ETA(epoch): 462s + [Epoch 92][30/40] loss=0.764441 avg=0.879360 VRAM=39.2GiB | 18.6% done | ETA(epoch): 231s + [Epoch 92][40/40] loss=0.859638 avg=0.878166 VRAM=39.0GiB | 18.6% done | ETA(epoch): 0s + Train loss: 0.878166 (923.8s) ETA: 6809min + Val loss: 0.888040 [t_0.0-0.2=1.0739 t_0.2-0.4=0.9843 t_0.4-0.6=0.8685 t_0.6-0.8=0.7089 t_0.8-1.0=0.6744] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 93/499 (19% done) --- + [Epoch 93][10/40] loss=0.888634 avg=0.862291 VRAM=39.2GiB | 18.6% done | ETA(epoch): 693s + [Epoch 93][20/40] loss=0.811261 avg=0.865084 VRAM=39.0GiB | 18.7% done | ETA(epoch): 462s + [Epoch 93][30/40] loss=0.888668 avg=0.860785 VRAM=39.2GiB | 18.8% done | ETA(epoch): 231s + [Epoch 93][40/40] loss=0.813850 avg=0.864786 VRAM=39.0GiB | 18.8% done | ETA(epoch): 0s + Train loss: 0.864786 (924.2s) ETA: 6791min + Val loss: 0.890523 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9810 t_0.4-0.6=0.8562 t_0.6-0.8=0.7255 t_0.8-1.0=0.6681] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 94/499 (19% done) --- + [Epoch 94][10/40] loss=0.801966 avg=0.848766 VRAM=39.2GiB | 18.9% done | ETA(epoch): 693s + [Epoch 94][20/40] loss=0.848664 avg=0.859742 VRAM=39.0GiB | 18.9% done | ETA(epoch): 462s + [Epoch 94][30/40] loss=0.769962 avg=0.871880 VRAM=39.2GiB | 18.9% done | ETA(epoch): 231s + [Epoch 94][40/40] loss=0.841007 avg=0.867269 VRAM=39.0GiB | 19.0% done | ETA(epoch): 0s + Train loss: 0.867269 (924.4s) ETA: 6773min + Val loss: 0.849769 [t_0.0-0.2=1.0651 t_0.2-0.4=1.0008 t_0.4-0.6=0.8137 t_0.6-0.8=0.7298 t_0.8-1.0=0.6856] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 95/499 (19% done) --- + [Epoch 95][10/40] loss=0.843492 avg=0.874933 VRAM=39.2GiB | 19.1% done | ETA(epoch): 693s + [Epoch 95][20/40] loss=0.792266 avg=0.861628 VRAM=39.0GiB | 19.1% done | ETA(epoch): 462s + [Epoch 95][30/40] loss=0.848397 avg=0.848785 VRAM=39.2GiB | 19.1% done | ETA(epoch): 231s + [Epoch 95][40/40] loss=0.925322 avg=0.851273 VRAM=39.0GiB | 19.2% done | ETA(epoch): 0s + Train loss: 0.851273 (924.1s) ETA: 6756min + Val loss: 0.858665 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9835 t_0.4-0.6=0.8351 t_0.6-0.8=0.7511 t_0.8-1.0=0.6741] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 96/499 (19% done) --- + [Epoch 96][10/40] loss=0.934894 avg=0.871844 VRAM=39.2GiB | 19.2% done | ETA(epoch): 693s + [Epoch 96][20/40] loss=0.795763 avg=0.878360 VRAM=39.0GiB | 19.3% done | ETA(epoch): 462s + [Epoch 96][30/40] loss=0.872184 avg=0.866596 VRAM=39.2GiB | 19.4% done | ETA(epoch): 231s + [Epoch 96][40/40] loss=0.802765 avg=0.862446 VRAM=39.0GiB | 19.4% done | ETA(epoch): 0s + Train loss: 0.862446 (924.3s) ETA: 6738min + Val loss: 0.859104 [t_0.0-0.2=1.0759 t_0.2-0.4=1.0123 t_0.4-0.6=0.8509 t_0.6-0.8=0.7140 t_0.8-1.0=0.6620] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 97/499 (19% done) --- + [Epoch 97][10/40] loss=0.888470 avg=0.895209 VRAM=39.2GiB | 19.4% done | ETA(epoch): 693s + [Epoch 97][20/40] loss=0.910198 avg=0.858156 VRAM=39.0GiB | 19.5% done | ETA(epoch): 462s + [Epoch 97][30/40] loss=0.861594 avg=0.862017 VRAM=39.2GiB | 19.6% done | ETA(epoch): 231s + [Epoch 97][40/40] loss=0.965783 avg=0.864462 VRAM=39.0GiB | 19.6% done | ETA(epoch): 0s + Train loss: 0.864462 (924.8s) ETA: 6721min + Val loss: 0.882773 [t_0.0-0.2=1.0659 t_0.2-0.4=1.0118 t_0.4-0.6=0.8540 t_0.6-0.8=0.7412 t_0.8-1.0=0.6728] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 98/499 (20% done) --- + [Epoch 98][10/40] loss=0.773542 avg=0.851700 VRAM=39.2GiB | 19.7% done | ETA(epoch): 694s + [Epoch 98][20/40] loss=0.747568 avg=0.845505 VRAM=39.0GiB | 19.7% done | ETA(epoch): 462s + [Epoch 98][30/40] loss=0.813144 avg=0.856255 VRAM=39.2GiB | 19.8% done | ETA(epoch): 231s + [Epoch 98][40/40] loss=0.830105 avg=0.859929 VRAM=39.0GiB | 19.8% done | ETA(epoch): 0s + Train loss: 0.859929 (924.4s) ETA: 6703min + Val loss: 0.856519 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9911 t_0.4-0.6=0.8480 t_0.6-0.8=0.7273 t_0.8-1.0=0.6510] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 99/499 (20% done) --- + [Epoch 99][10/40] loss=0.884363 avg=0.879838 VRAM=39.2GiB | 19.9% done | ETA(epoch): 693s + [Epoch 99][20/40] loss=0.906524 avg=0.876633 VRAM=39.0GiB | 19.9% done | ETA(epoch): 462s + [Epoch 99][30/40] loss=0.897772 avg=0.869039 VRAM=39.2GiB | 20.0% done | ETA(epoch): 231s + [Epoch 99][40/40] loss=0.746078 avg=0.855073 VRAM=39.0GiB | 20.0% done | ETA(epoch): 0s + Train loss: 0.855073 (924.7s) ETA: 6685min + Val loss: 0.890493 [t_0.0-0.2=1.0728 t_0.2-0.4=1.0301 t_0.4-0.6=0.8204 t_0.6-0.8=0.7076 t_0.8-1.0=0.6682] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 100/499 (20% done) --- + [MilestoneVis] train_0 step 4000 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 4000 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 4000 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 4000 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 4000 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 4000 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 4000 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 4000 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 4000 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 4000 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 100][10/40] loss=0.823015 avg=0.872037 VRAM=39.2GiB | 20.1% done | ETA(epoch): 2947s + [Epoch 100][20/40] loss=0.850622 avg=0.856041 VRAM=39.0GiB | 20.1% done | ETA(epoch): 1214s + [Epoch 100][30/40] loss=0.891251 avg=0.858652 VRAM=39.2GiB | 20.2% done | ETA(epoch): 482s + [Epoch 100][40/40] loss=0.887811 avg=0.856615 VRAM=39.0GiB | 20.2% done | ETA(epoch): 0s + Train loss: 0.856615 (1676.2s) ETA: 6717min + Val loss: 0.872851 [t_0.0-0.2=1.0659 t_0.2-0.4=1.0190 t_0.4-0.6=0.8765 t_0.6-0.8=0.7174 t_0.8-1.0=0.6646] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 101/499 (20% done) --- + [Epoch 101][10/40] loss=0.923325 avg=0.863052 VRAM=39.2GiB | 20.2% done | ETA(epoch): 693s + [Epoch 101][20/40] loss=0.821194 avg=0.855126 VRAM=39.0GiB | 20.3% done | ETA(epoch): 462s + [Epoch 101][30/40] loss=0.878540 avg=0.864031 VRAM=39.2GiB | 20.3% done | ETA(epoch): 231s + [Epoch 101][40/40] loss=0.780778 avg=0.861552 VRAM=39.0GiB | 20.4% done | ETA(epoch): 0s + Train loss: 0.861552 (924.1s) ETA: 6699min + Val loss: 0.852964 [t_0.0-0.2=1.0729 t_0.2-0.4=0.9743 t_0.4-0.6=0.8506 t_0.6-0.8=0.7226 t_0.8-1.0=0.6575] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 102/499 (20% done) --- + [Epoch 102][10/40] loss=0.795681 avg=0.854262 VRAM=39.2GiB | 20.4% done | ETA(epoch): 694s + [Epoch 102][20/40] loss=0.791284 avg=0.851066 VRAM=39.0GiB | 20.5% done | ETA(epoch): 462s + [Epoch 102][30/40] loss=0.874803 avg=0.834841 VRAM=39.2GiB | 20.5% done | ETA(epoch): 231s + [Epoch 102][40/40] loss=0.832285 avg=0.837564 VRAM=39.0GiB | 20.6% done | ETA(epoch): 0s + Train loss: 0.837564 (924.4s) ETA: 6681min + Val loss: 0.850985 [t_0.0-0.2=1.0540 t_0.2-0.4=0.9781 t_0.4-0.6=0.8610 t_0.6-0.8=0.7238 t_0.8-1.0=0.6489] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 103/499 (21% done) --- + [Epoch 103][10/40] loss=0.944255 avg=0.873924 VRAM=39.2GiB | 20.6% done | ETA(epoch): 693s + [Epoch 103][20/40] loss=0.899349 avg=0.857747 VRAM=39.0GiB | 20.7% done | ETA(epoch): 462s + [Epoch 103][30/40] loss=0.892687 avg=0.857066 VRAM=39.2GiB | 20.8% done | ETA(epoch): 231s + [Epoch 103][40/40] loss=0.922006 avg=0.858466 VRAM=39.0GiB | 20.8% done | ETA(epoch): 0s + Train loss: 0.858466 (923.6s) ETA: 6663min + Val loss: 0.897365 [t_0.0-0.2=1.0743 t_0.2-0.4=0.9954 t_0.4-0.6=0.8141 t_0.6-0.8=0.7476 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 104/499 (21% done) --- + [Epoch 104][10/40] loss=0.994319 avg=0.893250 VRAM=39.2GiB | 20.8% done | ETA(epoch): 693s + [Epoch 104][20/40] loss=0.799490 avg=0.885006 VRAM=39.0GiB | 20.9% done | ETA(epoch): 462s + [Epoch 104][30/40] loss=0.840226 avg=0.871060 VRAM=39.2GiB | 20.9% done | ETA(epoch): 231s + [Epoch 104][40/40] loss=0.797561 avg=0.868785 VRAM=39.0GiB | 21.0% done | ETA(epoch): 0s + Train loss: 0.868785 (923.9s) ETA: 6645min + Val loss: 0.834410 [t_0.0-0.2=1.0609 t_0.2-0.4=0.9437 t_0.4-0.6=0.8434 t_0.6-0.8=0.7364 t_0.8-1.0=0.6537] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 105/499 (21% done) --- + [Epoch 105][10/40] loss=0.882796 avg=0.883022 VRAM=39.2GiB | 21.1% done | ETA(epoch): 693s + [Epoch 105][20/40] loss=0.881807 avg=0.869122 VRAM=39.0GiB | 21.1% done | ETA(epoch): 462s + [Epoch 105][30/40] loss=0.883103 avg=0.861709 VRAM=39.2GiB | 21.1% done | ETA(epoch): 231s + [Epoch 105][40/40] loss=0.797117 avg=0.863949 VRAM=39.0GiB | 21.2% done | ETA(epoch): 0s + Train loss: 0.863949 (924.0s) ETA: 6627min + Val loss: 0.853981 [t_0.0-0.2=1.0807 t_0.2-0.4=0.9946 t_0.4-0.6=0.8518 t_0.6-0.8=0.6954 t_0.8-1.0=0.6610] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 106/499 (21% done) --- + [Epoch 106][10/40] loss=0.857255 avg=0.857060 VRAM=39.2GiB | 21.2% done | ETA(epoch): 692s + [Epoch 106][20/40] loss=0.879857 avg=0.884412 VRAM=39.0GiB | 21.3% done | ETA(epoch): 462s + [Epoch 106][30/40] loss=0.965709 avg=0.878386 VRAM=39.2GiB | 21.3% done | ETA(epoch): 231s + [Epoch 106][40/40] loss=0.827209 avg=0.878013 VRAM=39.0GiB | 21.4% done | ETA(epoch): 0s + Train loss: 0.878013 (923.6s) ETA: 6609min + Val loss: 0.876311 [t_0.0-0.2=1.0857 t_0.2-0.4=0.9804 t_0.4-0.6=0.8707 t_0.6-0.8=0.7022 t_0.8-1.0=0.6665] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 107/499 (21% done) --- + [Epoch 107][10/40] loss=0.787753 avg=0.844253 VRAM=39.2GiB | 21.4% done | ETA(epoch): 693s + [Epoch 107][20/40] loss=0.949731 avg=0.858491 VRAM=39.0GiB | 21.5% done | ETA(epoch): 462s + [Epoch 107][30/40] loss=0.779776 avg=0.859480 VRAM=39.2GiB | 21.6% done | ETA(epoch): 231s + [Epoch 107][40/40] loss=0.879724 avg=0.853425 VRAM=39.0GiB | 21.6% done | ETA(epoch): 0s + Train loss: 0.853425 (923.9s) ETA: 6591min + Val loss: 0.904209 [t_0.0-0.2=1.0733 t_0.2-0.4=0.9950 t_0.4-0.6=0.8679 t_0.6-0.8=0.7282 t_0.8-1.0=0.6678] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 108/499 (22% done) --- + [Epoch 108][10/40] loss=0.804495 avg=0.845366 VRAM=39.2GiB | 21.6% done | ETA(epoch): 693s + [Epoch 108][20/40] loss=0.867097 avg=0.837834 VRAM=39.0GiB | 21.7% done | ETA(epoch): 462s + [Epoch 108][30/40] loss=0.914060 avg=0.843828 VRAM=39.2GiB | 21.8% done | ETA(epoch): 231s + [Epoch 108][40/40] loss=0.763232 avg=0.838532 VRAM=39.0GiB | 21.8% done | ETA(epoch): 0s + Train loss: 0.838532 (924.3s) ETA: 6573min + Val loss: 0.847617 [t_0.0-0.2=1.0829 t_0.2-0.4=1.0076 t_0.4-0.6=0.8297 t_0.6-0.8=0.7165 t_0.8-1.0=0.6860] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 109/499 (22% done) --- + [Epoch 109][10/40] loss=0.901121 avg=0.858470 VRAM=39.2GiB | 21.9% done | ETA(epoch): 694s + [Epoch 109][20/40] loss=0.851722 avg=0.853204 VRAM=39.0GiB | 21.9% done | ETA(epoch): 462s + [Epoch 109][30/40] loss=0.881786 avg=0.861261 VRAM=39.2GiB | 21.9% done | ETA(epoch): 231s + [Epoch 109][40/40] loss=0.929192 avg=0.863250 VRAM=39.0GiB | 22.0% done | ETA(epoch): 0s + Train loss: 0.863250 (924.7s) ETA: 6555min + Val loss: 0.849374 [t_0.0-0.2=1.0640 t_0.2-0.4=1.0156 t_0.4-0.6=0.8225 t_0.6-0.8=0.7211 t_0.8-1.0=0.6615] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 110/499 (22% done) --- + [Epoch 110][10/40] loss=0.876022 avg=0.896437 VRAM=39.2GiB | 22.1% done | ETA(epoch): 692s + [Epoch 110][20/40] loss=0.900508 avg=0.876709 VRAM=39.0GiB | 22.1% done | ETA(epoch): 462s + [Epoch 110][30/40] loss=0.852849 avg=0.871688 VRAM=39.2GiB | 22.1% done | ETA(epoch): 231s + [Epoch 110][40/40] loss=0.867257 avg=0.868887 VRAM=39.0GiB | 22.2% done | ETA(epoch): 0s + Train loss: 0.868887 (924.5s) ETA: 6537min + Val loss: 0.863104 [t_0.0-0.2=1.0662 t_0.2-0.4=1.0029 t_0.4-0.6=0.8335 t_0.6-0.8=0.7492 t_0.8-1.0=0.6526] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 111/499 (22% done) --- + [Epoch 111][10/40] loss=0.802128 avg=0.846930 VRAM=39.2GiB | 22.2% done | ETA(epoch): 693s + [Epoch 111][20/40] loss=0.817245 avg=0.841730 VRAM=39.0GiB | 22.3% done | ETA(epoch): 462s + [Epoch 111][30/40] loss=0.827882 avg=0.845889 VRAM=39.2GiB | 22.4% done | ETA(epoch): 231s + [Epoch 111][40/40] loss=0.963339 avg=0.856902 VRAM=39.0GiB | 22.4% done | ETA(epoch): 0s + Train loss: 0.856902 (924.4s) ETA: 6519min + Val loss: 0.860933 [t_0.0-0.2=1.0665 t_0.2-0.4=0.9752 t_0.4-0.6=0.8515 t_0.6-0.8=0.7069 t_0.8-1.0=0.6633] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 112/499 (22% done) --- + [Epoch 112][10/40] loss=0.858448 avg=0.879942 VRAM=39.2GiB | 22.4% done | ETA(epoch): 693s + [Epoch 112][20/40] loss=0.932506 avg=0.857809 VRAM=39.0GiB | 22.5% done | ETA(epoch): 462s + [Epoch 112][30/40] loss=0.785083 avg=0.853353 VRAM=39.2GiB | 22.6% done | ETA(epoch): 231s + [Epoch 112][40/40] loss=0.881558 avg=0.851592 VRAM=39.0GiB | 22.6% done | ETA(epoch): 0s + Train loss: 0.851592 (924.1s) ETA: 6502min + Val loss: 0.869313 [t_0.0-0.2=1.0747 t_0.2-0.4=1.0284 t_0.4-0.6=0.8206 t_0.6-0.8=0.7030 t_0.8-1.0=0.6546] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 18.5/188.4 GiB (9.8%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 113/499 (23% done) --- + [Epoch 113][10/40] loss=0.791427 avg=0.846929 VRAM=39.2GiB | 22.7% done | ETA(epoch): 693s + [Epoch 113][20/40] loss=0.845194 avg=0.853751 VRAM=39.0GiB | 22.7% done | ETA(epoch): 462s + [Epoch 113][30/40] loss=0.899540 avg=0.866300 VRAM=39.2GiB | 22.8% done | ETA(epoch): 231s + [Epoch 113][40/40] loss=0.863734 avg=0.865360 VRAM=39.0GiB | 22.8% done | ETA(epoch): 0s + Train loss: 0.865360 (924.2s) ETA: 6484min + Val loss: 0.842820 [t_0.0-0.2=1.0604 t_0.2-0.4=0.9947 t_0.4-0.6=0.8537 t_0.6-0.8=0.7335 t_0.8-1.0=0.6585] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 114/499 (23% done) --- + [Epoch 114][10/40] loss=0.835548 avg=0.869539 VRAM=39.2GiB | 22.9% done | ETA(epoch): 692s + [Epoch 114][20/40] loss=0.892645 avg=0.870643 VRAM=39.0GiB | 22.9% done | ETA(epoch): 462s + [Epoch 114][30/40] loss=0.771210 avg=0.867460 VRAM=39.2GiB | 22.9% done | ETA(epoch): 231s + [Epoch 114][40/40] loss=0.763607 avg=0.855253 VRAM=39.0GiB | 23.0% done | ETA(epoch): 0s + Train loss: 0.855253 (923.4s) ETA: 6466min + Val loss: 0.882909 [t_0.0-0.2=1.0738 t_0.2-0.4=1.0240 t_0.4-0.6=0.8396 t_0.6-0.8=0.6961 t_0.8-1.0=0.6563] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 115/499 (23% done) --- + [Epoch 115][10/40] loss=0.832272 avg=0.861044 VRAM=39.2GiB | 23.1% done | ETA(epoch): 693s + [Epoch 115][20/40] loss=0.923389 avg=0.865460 VRAM=39.0GiB | 23.1% done | ETA(epoch): 462s + [Epoch 115][30/40] loss=0.913373 avg=0.855470 VRAM=39.2GiB | 23.2% done | ETA(epoch): 231s + [Epoch 115][40/40] loss=0.961264 avg=0.863897 VRAM=39.0GiB | 23.2% done | ETA(epoch): 0s + Train loss: 0.863897 (923.8s) ETA: 6448min + Val loss: 0.848247 [t_0.0-0.2=1.0849 t_0.2-0.4=0.9867 t_0.4-0.6=0.8203 t_0.6-0.8=0.7304 t_0.8-1.0=0.6642] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 116/499 (23% done) --- + [Epoch 116][10/40] loss=0.896813 avg=0.859299 VRAM=39.2GiB | 23.2% done | ETA(epoch): 693s + [Epoch 116][20/40] loss=0.829752 avg=0.851086 VRAM=39.0GiB | 23.3% done | ETA(epoch): 462s + [Epoch 116][30/40] loss=0.930071 avg=0.856876 VRAM=39.2GiB | 23.4% done | ETA(epoch): 231s + [Epoch 116][40/40] loss=0.962060 avg=0.865243 VRAM=39.0GiB | 23.4% done | ETA(epoch): 0s + Train loss: 0.865243 (924.4s) ETA: 6430min + Val loss: 0.847634 [t_0.0-0.2=1.0866 t_0.2-0.4=1.0061 t_0.4-0.6=0.8569 t_0.6-0.8=0.7131 t_0.8-1.0=0.6492] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 117/499 (23% done) --- + [Epoch 117][10/40] loss=0.768134 avg=0.849851 VRAM=39.2GiB | 23.4% done | ETA(epoch): 693s + [Epoch 117][20/40] loss=0.788285 avg=0.833166 VRAM=39.0GiB | 23.5% done | ETA(epoch): 462s + [Epoch 117][30/40] loss=0.971608 avg=0.842746 VRAM=39.2GiB | 23.5% done | ETA(epoch): 231s + [Epoch 117][40/40] loss=0.856482 avg=0.842738 VRAM=39.0GiB | 23.6% done | ETA(epoch): 0s + Train loss: 0.842738 (924.3s) ETA: 6413min + Val loss: 0.838888 [t_0.0-0.2=1.0705 t_0.2-0.4=1.0001 t_0.4-0.6=0.8542 t_0.6-0.8=0.7075 t_0.8-1.0=0.6497] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 118/499 (24% done) --- + [Epoch 118][10/40] loss=0.833889 avg=0.864839 VRAM=39.2GiB | 23.6% done | ETA(epoch): 692s + [Epoch 118][20/40] loss=0.798115 avg=0.855126 VRAM=39.0GiB | 23.7% done | ETA(epoch): 462s + [Epoch 118][30/40] loss=1.023712 avg=0.850609 VRAM=39.2GiB | 23.8% done | ETA(epoch): 231s + [Epoch 118][40/40] loss=0.826707 avg=0.850704 VRAM=39.0GiB | 23.8% done | ETA(epoch): 0s + Train loss: 0.850704 (923.4s) ETA: 6395min + Val loss: 0.881028 [t_0.0-0.2=1.0726 t_0.2-0.4=1.0105 t_0.4-0.6=0.8322 t_0.6-0.8=0.7179 t_0.8-1.0=0.6537] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 119/499 (24% done) --- + [Epoch 119][10/40] loss=0.881217 avg=0.867649 VRAM=39.2GiB | 23.8% done | ETA(epoch): 692s + [Epoch 119][20/40] loss=0.867510 avg=0.879827 VRAM=39.0GiB | 23.9% done | ETA(epoch): 462s + [Epoch 119][30/40] loss=0.876585 avg=0.875648 VRAM=39.2GiB | 23.9% done | ETA(epoch): 231s + [Epoch 119][40/40] loss=0.865921 avg=0.871981 VRAM=39.0GiB | 24.0% done | ETA(epoch): 0s + Train loss: 0.871981 (924.0s) ETA: 6377min + Val loss: 0.867136 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9699 t_0.4-0.6=0.8346 t_0.6-0.8=0.7186 t_0.8-1.0=0.6604] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 120/499 (24% done) --- + [Epoch 120][10/40] loss=0.876276 avg=0.854944 VRAM=39.2GiB | 24.1% done | ETA(epoch): 693s + [Epoch 120][20/40] loss=0.788217 avg=0.854019 VRAM=39.0GiB | 24.1% done | ETA(epoch): 462s + [Epoch 120][30/40] loss=0.824860 avg=0.848395 VRAM=39.2GiB | 24.1% done | ETA(epoch): 231s + [Epoch 120][40/40] loss=0.864902 avg=0.851851 VRAM=39.0GiB | 24.2% done | ETA(epoch): 0s + Train loss: 0.851851 (924.5s) ETA: 6360min + Val loss: 0.845354 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9854 t_0.4-0.6=0.8538 t_0.6-0.8=0.7214 t_0.8-1.0=0.6517] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 121/499 (24% done) --- + [Epoch 121][10/40] loss=0.820773 avg=0.857512 VRAM=39.2GiB | 24.2% done | ETA(epoch): 694s + [Epoch 121][20/40] loss=0.758440 avg=0.848171 VRAM=39.0GiB | 24.3% done | ETA(epoch): 463s + [Epoch 121][30/40] loss=0.767788 avg=0.841187 VRAM=39.2GiB | 24.3% done | ETA(epoch): 231s + [Epoch 121][40/40] loss=0.866962 avg=0.845464 VRAM=39.0GiB | 24.4% done | ETA(epoch): 0s + Train loss: 0.845464 (925.0s) ETA: 6342min + Val loss: 0.813085 [t_0.0-0.2=1.0641 t_0.2-0.4=0.9755 t_0.4-0.6=0.7981 t_0.6-0.8=0.7066 t_0.8-1.0=0.6615] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0121 (BEST) + Deleted old checkpoint: checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 122/499 (24% done) --- + [Epoch 122][10/40] loss=0.859176 avg=0.842218 VRAM=39.2GiB | 24.4% done | ETA(epoch): 693s + [Epoch 122][20/40] loss=0.829996 avg=0.851135 VRAM=39.0GiB | 24.5% done | ETA(epoch): 462s + [Epoch 122][30/40] loss=0.874461 avg=0.857146 VRAM=39.2GiB | 24.6% done | ETA(epoch): 231s + [Epoch 122][40/40] loss=0.839283 avg=0.858228 VRAM=39.0GiB | 24.6% done | ETA(epoch): 0s + Train loss: 0.858228 (924.3s) ETA: 6324min + Val loss: 0.857793 [t_0.0-0.2=1.0556 t_0.2-0.4=1.0017 t_0.4-0.6=0.8081 t_0.6-0.8=0.7168 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 123/499 (25% done) --- + [Epoch 123][10/40] loss=0.872579 avg=0.862996 VRAM=39.2GiB | 24.6% done | ETA(epoch): 693s + [Epoch 123][20/40] loss=0.898370 avg=0.867211 VRAM=39.0GiB | 24.7% done | ETA(epoch): 462s + [Epoch 123][30/40] loss=0.923199 avg=0.868467 VRAM=39.2GiB | 24.8% done | ETA(epoch): 231s + [Epoch 123][40/40] loss=0.914787 avg=0.862013 VRAM=39.0GiB | 24.8% done | ETA(epoch): 0s + Train loss: 0.862013 (924.6s) ETA: 6307min + Val loss: 0.883198 [t_0.0-0.2=1.0764 t_0.2-0.4=0.9944 t_0.4-0.6=0.8427 t_0.6-0.8=0.7155 t_0.8-1.0=0.6357] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 124/499 (25% done) --- + [Epoch 124][10/40] loss=0.798577 avg=0.834712 VRAM=39.2GiB | 24.9% done | ETA(epoch): 693s + [Epoch 124][20/40] loss=0.872180 avg=0.845854 VRAM=39.0GiB | 24.9% done | ETA(epoch): 462s + [Epoch 124][30/40] loss=0.816434 avg=0.847004 VRAM=39.2GiB | 24.9% done | ETA(epoch): 231s + [Epoch 124][40/40] loss=0.927156 avg=0.854739 VRAM=39.0GiB | 25.0% done | ETA(epoch): 0s + Train loss: 0.854739 (924.6s) ETA: 6289min + Val loss: 0.858648 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0098 t_0.4-0.6=0.8308 t_0.6-0.8=0.7303 t_0.8-1.0=0.6720] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0124 +[MEM @ epoch 124 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 125/499 (25% done) --- + [Epoch 125][10/40] loss=0.814910 avg=0.845170 VRAM=39.2GiB | 25.1% done | ETA(epoch): 693s + [Epoch 125][20/40] loss=0.866427 avg=0.866111 VRAM=39.0GiB | 25.1% done | ETA(epoch): 462s + [Epoch 125][30/40] loss=0.791751 avg=0.854379 VRAM=39.2GiB | 25.1% done | ETA(epoch): 231s + [Epoch 125][40/40] loss=0.801823 avg=0.860948 VRAM=39.0GiB | 25.2% done | ETA(epoch): 0s + Train loss: 0.860948 (925.0s) ETA: 6272min + Val loss: 0.881735 [t_0.0-0.2=1.0701 t_0.2-0.4=1.0045 t_0.4-0.6=0.8222 t_0.6-0.8=0.7371 t_0.8-1.0=0.6769] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 126/499 (25% done) --- + [Epoch 126][10/40] loss=0.909425 avg=0.888975 VRAM=39.2GiB | 25.2% done | ETA(epoch): 693s + [Epoch 126][20/40] loss=0.843091 avg=0.867196 VRAM=39.0GiB | 25.3% done | ETA(epoch): 462s + [Epoch 126][30/40] loss=0.913691 avg=0.863159 VRAM=39.2GiB | 25.4% done | ETA(epoch): 231s + [Epoch 126][40/40] loss=0.802681 avg=0.857065 VRAM=39.0GiB | 25.4% done | ETA(epoch): 0s + Train loss: 0.857065 (924.7s) ETA: 6254min + Val loss: 0.847671 [t_0.0-0.2=1.0799 t_0.2-0.4=1.0103 t_0.4-0.6=0.8092 t_0.6-0.8=0.7162 t_0.8-1.0=0.6673] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 127/499 (25% done) --- + [Epoch 127][10/40] loss=0.785066 avg=0.833151 VRAM=39.2GiB | 25.4% done | ETA(epoch): 693s + [Epoch 127][20/40] loss=0.850120 avg=0.848887 VRAM=39.0GiB | 25.5% done | ETA(epoch): 463s + [Epoch 127][30/40] loss=0.902188 avg=0.854888 VRAM=39.2GiB | 25.6% done | ETA(epoch): 231s + [Epoch 127][40/40] loss=0.805519 avg=0.847734 VRAM=39.0GiB | 25.6% done | ETA(epoch): 0s + Train loss: 0.847734 (925.2s) ETA: 6237min + Val loss: 0.830031 [t_0.0-0.2=1.0717 t_0.2-0.4=0.9419 t_0.4-0.6=0.8846 t_0.6-0.8=0.7180 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 128/499 (26% done) --- + [Epoch 128][10/40] loss=0.911962 avg=0.845537 VRAM=39.2GiB | 25.7% done | ETA(epoch): 694s + [Epoch 128][20/40] loss=0.907589 avg=0.851807 VRAM=39.0GiB | 25.7% done | ETA(epoch): 463s + [Epoch 128][30/40] loss=0.839002 avg=0.845835 VRAM=39.2GiB | 25.8% done | ETA(epoch): 231s + [Epoch 128][40/40] loss=0.812341 avg=0.849757 VRAM=39.0GiB | 25.8% done | ETA(epoch): 0s + Train loss: 0.849757 (925.3s) ETA: 6219min + Val loss: 0.831985 [t_0.0-0.2=1.0627 t_0.2-0.4=0.9665 t_0.4-0.6=0.8609 t_0.6-0.8=0.7098 t_0.8-1.0=0.6585] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 129/499 (26% done) --- + [Epoch 129][10/40] loss=0.781215 avg=0.852512 VRAM=39.2GiB | 25.9% done | ETA(epoch): 694s + [Epoch 129][20/40] loss=0.844765 avg=0.854027 VRAM=39.0GiB | 25.9% done | ETA(epoch): 462s + [Epoch 129][30/40] loss=0.752290 avg=0.856044 VRAM=39.2GiB | 25.9% done | ETA(epoch): 231s + [Epoch 129][40/40] loss=0.850230 avg=0.859991 VRAM=39.0GiB | 26.0% done | ETA(epoch): 0s + Train loss: 0.859991 (925.0s) ETA: 6202min + Val loss: 0.856040 [t_0.0-0.2=1.0730 t_0.2-0.4=1.0114 t_0.4-0.6=0.8433 t_0.6-0.8=0.7307 t_0.8-1.0=0.6591] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 130/499 (26% done) --- + [Epoch 130][10/40] loss=0.834636 avg=0.815526 VRAM=39.2GiB | 26.1% done | ETA(epoch): 693s + [Epoch 130][20/40] loss=0.874368 avg=0.828296 VRAM=39.0GiB | 26.1% done | ETA(epoch): 463s + [Epoch 130][30/40] loss=0.949441 avg=0.842229 VRAM=39.2GiB | 26.2% done | ETA(epoch): 231s + [Epoch 130][40/40] loss=0.844462 avg=0.843388 VRAM=39.0GiB | 26.2% done | ETA(epoch): 0s + Train loss: 0.843388 (925.2s) ETA: 6184min + Val loss: 0.873406 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9922 t_0.4-0.6=0.8807 t_0.6-0.8=0.6961 t_0.8-1.0=0.6608] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 131/499 (26% done) --- + [Epoch 131][10/40] loss=0.766458 avg=0.854698 VRAM=39.2GiB | 26.2% done | ETA(epoch): 693s + [Epoch 131][20/40] loss=0.793645 avg=0.851695 VRAM=39.0GiB | 26.3% done | ETA(epoch): 463s + [Epoch 131][30/40] loss=0.753586 avg=0.852597 VRAM=39.2GiB | 26.4% done | ETA(epoch): 231s + [Epoch 131][40/40] loss=0.712360 avg=0.846158 VRAM=39.0GiB | 26.4% done | ETA(epoch): 0s + Train loss: 0.846158 (925.0s) ETA: 6167min + Val loss: 0.831869 [t_0.0-0.2=1.0708 t_0.2-0.4=0.9963 t_0.4-0.6=0.8645 t_0.6-0.8=0.7276 t_0.8-1.0=0.6657] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 132/499 (26% done) --- + [Epoch 132][10/40] loss=0.875793 avg=0.859606 VRAM=39.2GiB | 26.5% done | ETA(epoch): 694s + [Epoch 132][20/40] loss=0.809119 avg=0.851871 VRAM=39.0GiB | 26.5% done | ETA(epoch): 463s + [Epoch 132][30/40] loss=0.825215 avg=0.846204 VRAM=39.2GiB | 26.6% done | ETA(epoch): 231s + [Epoch 132][40/40] loss=0.891727 avg=0.858140 VRAM=39.0GiB | 26.6% done | ETA(epoch): 0s + Train loss: 0.858140 (925.5s) ETA: 6150min + Val loss: 0.836278 [t_0.0-0.2=1.0644 t_0.2-0.4=0.9706 t_0.4-0.6=0.8056 t_0.6-0.8=0.7337 t_0.8-1.0=0.6532] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 133/499 (27% done) --- + [Epoch 133][10/40] loss=0.814203 avg=0.833131 VRAM=39.2GiB | 26.7% done | ETA(epoch): 694s + [Epoch 133][20/40] loss=0.849507 avg=0.844056 VRAM=39.0GiB | 26.7% done | ETA(epoch): 463s + [Epoch 133][30/40] loss=0.852624 avg=0.855815 VRAM=39.2GiB | 26.8% done | ETA(epoch): 231s + [Epoch 133][40/40] loss=0.791803 avg=0.863580 VRAM=39.0GiB | 26.8% done | ETA(epoch): 0s + Train loss: 0.863580 (925.7s) ETA: 6132min + Val loss: 0.862634 [t_0.0-0.2=1.0912 t_0.2-0.4=0.9976 t_0.4-0.6=0.8239 t_0.6-0.8=0.7257 t_0.8-1.0=0.6478] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 134/499 (27% done) --- + [Epoch 134][10/40] loss=0.847143 avg=0.861852 VRAM=39.2GiB | 26.9% done | ETA(epoch): 694s + [Epoch 134][20/40] loss=0.870544 avg=0.861795 VRAM=39.0GiB | 26.9% done | ETA(epoch): 463s + [Epoch 134][30/40] loss=0.823163 avg=0.858726 VRAM=39.2GiB | 27.0% done | ETA(epoch): 231s + [Epoch 134][40/40] loss=0.999907 avg=0.859384 VRAM=39.0GiB | 27.0% done | ETA(epoch): 0s + Train loss: 0.859384 (925.2s) ETA: 6115min + Val loss: 0.878350 [t_0.0-0.2=1.0838 t_0.2-0.4=0.9818 t_0.4-0.6=0.8462 t_0.6-0.8=0.7170 t_0.8-1.0=0.6506] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 135/499 (27% done) --- + [Epoch 135][10/40] loss=0.828116 avg=0.837706 VRAM=39.2GiB | 27.1% done | ETA(epoch): 694s + [Epoch 135][20/40] loss=0.836588 avg=0.850612 VRAM=39.0GiB | 27.1% done | ETA(epoch): 463s + [Epoch 135][30/40] loss=0.872953 avg=0.852156 VRAM=39.2GiB | 27.2% done | ETA(epoch): 231s + [Epoch 135][40/40] loss=0.808228 avg=0.858280 VRAM=39.0GiB | 27.2% done | ETA(epoch): 0s + Train loss: 0.858280 (925.5s) ETA: 6097min + Val loss: 0.863173 [t_0.0-0.2=1.0651 t_0.2-0.4=0.9800 t_0.4-0.6=0.8662 t_0.6-0.8=0.7127 t_0.8-1.0=0.6630] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 136/499 (27% done) --- + [Epoch 136][10/40] loss=0.726569 avg=0.833338 VRAM=39.2GiB | 27.3% done | ETA(epoch): 694s + [Epoch 136][20/40] loss=0.946687 avg=0.842982 VRAM=39.0GiB | 27.3% done | ETA(epoch): 463s + [Epoch 136][30/40] loss=0.921803 avg=0.846232 VRAM=39.2GiB | 27.4% done | ETA(epoch): 231s + [Epoch 136][40/40] loss=0.956890 avg=0.847591 VRAM=39.0GiB | 27.4% done | ETA(epoch): 0s + Train loss: 0.847591 (925.4s) ETA: 6080min + Val loss: 0.878630 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9926 t_0.4-0.6=0.8360 t_0.6-0.8=0.7171 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 137/499 (27% done) --- + [Epoch 137][10/40] loss=0.836908 avg=0.833864 VRAM=39.2GiB | 27.5% done | ETA(epoch): 694s + [Epoch 137][20/40] loss=0.793626 avg=0.842975 VRAM=39.0GiB | 27.5% done | ETA(epoch): 463s + [Epoch 137][30/40] loss=0.755142 avg=0.853132 VRAM=39.2GiB | 27.6% done | ETA(epoch): 231s + [Epoch 137][40/40] loss=0.879691 avg=0.857218 VRAM=39.0GiB | 27.6% done | ETA(epoch): 0s + Train loss: 0.857218 (925.3s) ETA: 6063min + Val loss: 0.888664 [t_0.0-0.2=1.0875 t_0.2-0.4=1.0068 t_0.4-0.6=0.8064 t_0.6-0.8=0.7054 t_0.8-1.0=0.6637] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 138/499 (28% done) --- + [Epoch 138][10/40] loss=0.876912 avg=0.853324 VRAM=39.2GiB | 27.7% done | ETA(epoch): 694s + [Epoch 138][20/40] loss=0.809267 avg=0.867198 VRAM=39.0GiB | 27.7% done | ETA(epoch): 463s + [Epoch 138][30/40] loss=0.893831 avg=0.858561 VRAM=39.2GiB | 27.8% done | ETA(epoch): 231s + [Epoch 138][40/40] loss=0.709963 avg=0.849025 VRAM=39.0GiB | 27.8% done | ETA(epoch): 0s + Train loss: 0.849025 (925.5s) ETA: 6045min + Val loss: 0.833635 [t_0.0-0.2=1.0549 t_0.2-0.4=1.0274 t_0.4-0.6=0.8522 t_0.6-0.8=0.7099 t_0.8-1.0=0.6533] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 139/499 (28% done) --- + [Epoch 139][10/40] loss=0.913215 avg=0.870620 VRAM=39.2GiB | 27.9% done | ETA(epoch): 694s + [Epoch 139][20/40] loss=0.898589 avg=0.858683 VRAM=39.0GiB | 27.9% done | ETA(epoch): 463s + [Epoch 139][30/40] loss=0.855137 avg=0.867272 VRAM=39.2GiB | 28.0% done | ETA(epoch): 231s + [Epoch 139][40/40] loss=0.943983 avg=0.864919 VRAM=39.0GiB | 28.0% done | ETA(epoch): 0s + Train loss: 0.864919 (925.1s) ETA: 6028min + Val loss: 0.860367 [t_0.0-0.2=1.0744 t_0.2-0.4=0.9923 t_0.4-0.6=0.8413 t_0.6-0.8=0.7388 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 140/499 (28% done) --- + [Epoch 140][10/40] loss=0.897682 avg=0.861898 VRAM=39.2GiB | 28.1% done | ETA(epoch): 694s + [Epoch 140][20/40] loss=0.860193 avg=0.851207 VRAM=39.0GiB | 28.1% done | ETA(epoch): 463s + [Epoch 140][30/40] loss=0.805770 avg=0.848663 VRAM=39.2GiB | 28.1% done | ETA(epoch): 231s + [Epoch 140][40/40] loss=0.914322 avg=0.846019 VRAM=39.0GiB | 28.2% done | ETA(epoch): 0s + Train loss: 0.846019 (925.1s) ETA: 6011min + Val loss: 0.830410 [t_0.0-0.2=1.0846 t_0.2-0.4=0.9968 t_0.4-0.6=0.8390 t_0.6-0.8=0.7128 t_0.8-1.0=0.6407] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 141/499 (28% done) --- + [Epoch 141][10/40] loss=0.876132 avg=0.868510 VRAM=39.2GiB | 28.2% done | ETA(epoch): 693s + [Epoch 141][20/40] loss=0.788208 avg=0.861657 VRAM=39.0GiB | 28.3% done | ETA(epoch): 462s + [Epoch 141][30/40] loss=0.889762 avg=0.859896 VRAM=39.2GiB | 28.3% done | ETA(epoch): 231s + [Epoch 141][40/40] loss=0.799661 avg=0.857101 VRAM=39.0GiB | 28.4% done | ETA(epoch): 0s + Train loss: 0.857101 (925.0s) ETA: 5993min + Val loss: 0.871827 [t_0.0-0.2=1.0699 t_0.2-0.4=1.0067 t_0.4-0.6=0.8850 t_0.6-0.8=0.6993 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 142/499 (28% done) --- + [Epoch 142][10/40] loss=0.769729 avg=0.819220 VRAM=39.2GiB | 28.4% done | ETA(epoch): 693s + [Epoch 142][20/40] loss=0.780291 avg=0.821760 VRAM=39.0GiB | 28.5% done | ETA(epoch): 462s + [Epoch 142][30/40] loss=0.888512 avg=0.838223 VRAM=39.2GiB | 28.5% done | ETA(epoch): 231s + [Epoch 142][40/40] loss=1.005126 avg=0.849498 VRAM=39.0GiB | 28.6% done | ETA(epoch): 0s + Train loss: 0.849498 (924.5s) ETA: 5976min + Val loss: 0.834571 [t_0.0-0.2=1.0651 t_0.2-0.4=1.0063 t_0.4-0.6=0.8344 t_0.6-0.8=0.7198 t_0.8-1.0=0.6429] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 143/499 (29% done) --- + [Epoch 143][10/40] loss=0.904160 avg=0.862791 VRAM=39.2GiB | 28.6% done | ETA(epoch): 693s + [Epoch 143][20/40] loss=0.883174 avg=0.869568 VRAM=39.0GiB | 28.7% done | ETA(epoch): 462s + [Epoch 143][30/40] loss=0.800304 avg=0.860433 VRAM=39.2GiB | 28.7% done | ETA(epoch): 231s + [Epoch 143][40/40] loss=0.830627 avg=0.858602 VRAM=39.0GiB | 28.8% done | ETA(epoch): 0s + Train loss: 0.858602 (924.7s) ETA: 5959min + Val loss: 0.854894 [t_0.0-0.2=1.0758 t_0.2-0.4=0.9805 t_0.4-0.6=0.8281 t_0.6-0.8=0.7230 t_0.8-1.0=0.6562] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 144/499 (29% done) --- + [Epoch 144][10/40] loss=0.800517 avg=0.865247 VRAM=39.2GiB | 28.8% done | ETA(epoch): 693s + [Epoch 144][20/40] loss=0.966065 avg=0.864757 VRAM=39.0GiB | 28.9% done | ETA(epoch): 462s + [Epoch 144][30/40] loss=0.769764 avg=0.857277 VRAM=39.2GiB | 28.9% done | ETA(epoch): 231s + [Epoch 144][40/40] loss=0.758465 avg=0.854643 VRAM=39.0GiB | 29.0% done | ETA(epoch): 0s + Train loss: 0.854643 (924.5s) ETA: 5942min + Val loss: 0.840549 [t_0.0-0.2=1.0523 t_0.2-0.4=0.9643 t_0.4-0.6=0.8533 t_0.6-0.8=0.7385 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 145/499 (29% done) --- + [Epoch 145][10/40] loss=0.840103 avg=0.886146 VRAM=39.2GiB | 29.0% done | ETA(epoch): 693s + [Epoch 145][20/40] loss=0.824367 avg=0.870431 VRAM=39.0GiB | 29.1% done | ETA(epoch): 462s + [Epoch 145][30/40] loss=0.801846 avg=0.864924 VRAM=39.2GiB | 29.1% done | ETA(epoch): 231s + [Epoch 145][40/40] loss=0.836489 avg=0.860952 VRAM=39.0GiB | 29.2% done | ETA(epoch): 0s + Train loss: 0.860952 (924.8s) ETA: 5924min + Val loss: 0.876367 [t_0.0-0.2=1.0685 t_0.2-0.4=0.9910 t_0.4-0.6=0.8202 t_0.6-0.8=0.7545 t_0.8-1.0=0.6310] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 146/499 (29% done) --- + [Epoch 146][10/40] loss=0.855392 avg=0.862063 VRAM=39.2GiB | 29.2% done | ETA(epoch): 694s + [Epoch 146][20/40] loss=0.890775 avg=0.862924 VRAM=39.0GiB | 29.3% done | ETA(epoch): 462s + [Epoch 146][30/40] loss=0.836429 avg=0.865917 VRAM=39.2GiB | 29.3% done | ETA(epoch): 231s + [Epoch 146][40/40] loss=0.953100 avg=0.860280 VRAM=39.0GiB | 29.4% done | ETA(epoch): 0s + Train loss: 0.860280 (924.6s) ETA: 5907min + Val loss: 0.877868 [t_0.0-0.2=1.0652 t_0.2-0.4=1.0210 t_0.4-0.6=0.8343 t_0.6-0.8=0.7172 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 147/499 (29% done) --- + [Epoch 147][10/40] loss=0.906339 avg=0.867670 VRAM=39.2GiB | 29.4% done | ETA(epoch): 693s + [Epoch 147][20/40] loss=0.922606 avg=0.862232 VRAM=39.0GiB | 29.5% done | ETA(epoch): 462s + [Epoch 147][30/40] loss=0.899042 avg=0.860810 VRAM=39.2GiB | 29.5% done | ETA(epoch): 231s + [Epoch 147][40/40] loss=0.808621 avg=0.864558 VRAM=39.0GiB | 29.6% done | ETA(epoch): 0s + Train loss: 0.864558 (924.2s) ETA: 5890min + Val loss: 0.840355 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9699 t_0.4-0.6=0.8303 t_0.6-0.8=0.7368 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 148/499 (30% done) --- + [Epoch 148][10/40] loss=0.803269 avg=0.859879 VRAM=39.2GiB | 29.6% done | ETA(epoch): 693s + [Epoch 148][20/40] loss=0.849521 avg=0.857573 VRAM=39.0GiB | 29.7% done | ETA(epoch): 462s + [Epoch 148][30/40] loss=0.867303 avg=0.856074 VRAM=39.2GiB | 29.8% done | ETA(epoch): 231s + [Epoch 148][40/40] loss=0.786067 avg=0.852901 VRAM=39.0GiB | 29.8% done | ETA(epoch): 0s + Train loss: 0.852901 (924.7s) ETA: 5872min + Val loss: 0.855576 [t_0.0-0.2=1.0942 t_0.2-0.4=0.9620 t_0.4-0.6=0.8257 t_0.6-0.8=0.7240 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 149/499 (30% done) --- + [Epoch 149][10/40] loss=0.942864 avg=0.852600 VRAM=39.2GiB | 29.8% done | ETA(epoch): 693s + [Epoch 149][20/40] loss=0.956977 avg=0.850935 VRAM=39.0GiB | 29.9% done | ETA(epoch): 462s + [Epoch 149][30/40] loss=0.784819 avg=0.860651 VRAM=39.2GiB | 29.9% done | ETA(epoch): 231s + [Epoch 149][40/40] loss=0.902515 avg=0.863369 VRAM=39.0GiB | 30.0% done | ETA(epoch): 0s + Train loss: 0.863369 (924.7s) ETA: 5855min + Val loss: 0.849909 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9874 t_0.4-0.6=0.8720 t_0.6-0.8=0.6950 t_0.8-1.0=0.6288] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 150/499 (30% done) --- + [MilestoneVis] train_0 step 6000 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 6000 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 6000 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 6000 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 6000 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 6000 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 6000 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 6000 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 6000 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 6000 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/40] loss=0.885721 avg=0.845223 VRAM=39.2GiB | 30.0% done | ETA(epoch): 2950s + [Epoch 150][20/40] loss=0.829819 avg=0.845998 VRAM=39.0GiB | 30.1% done | ETA(epoch): 1214s + [Epoch 150][30/40] loss=0.865691 avg=0.852220 VRAM=39.2GiB | 30.1% done | ETA(epoch): 482s + [Epoch 150][40/40] loss=0.968547 avg=0.853654 VRAM=39.0GiB | 30.2% done | ETA(epoch): 0s + Train loss: 0.853654 (1677.0s) ETA: 5867min + Val loss: 0.875384 [t_0.0-0.2=1.0809 t_0.2-0.4=1.0192 t_0.4-0.6=0.8174 t_0.6-0.8=0.7314 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 151/499 (30% done) --- + [Epoch 151][10/40] loss=0.960516 avg=0.885134 VRAM=39.2GiB | 30.2% done | ETA(epoch): 693s + [Epoch 151][20/40] loss=0.921982 avg=0.867829 VRAM=39.0GiB | 30.3% done | ETA(epoch): 462s + [Epoch 151][30/40] loss=0.951125 avg=0.864374 VRAM=39.2GiB | 30.3% done | ETA(epoch): 231s + [Epoch 151][40/40] loss=0.842295 avg=0.867195 VRAM=39.0GiB | 30.4% done | ETA(epoch): 0s + Train loss: 0.867195 (925.0s) ETA: 5849min + Val loss: 0.855251 [t_0.0-0.2=1.0614 t_0.2-0.4=1.0326 t_0.4-0.6=0.8707 t_0.6-0.8=0.7044 t_0.8-1.0=0.6589] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 152/499 (30% done) --- + [Epoch 152][10/40] loss=0.892835 avg=0.859236 VRAM=39.2GiB | 30.4% done | ETA(epoch): 693s + [Epoch 152][20/40] loss=0.828076 avg=0.847110 VRAM=39.0GiB | 30.5% done | ETA(epoch): 462s + [Epoch 152][30/40] loss=0.872270 avg=0.840856 VRAM=39.2GiB | 30.6% done | ETA(epoch): 231s + [Epoch 152][40/40] loss=0.842104 avg=0.843176 VRAM=39.0GiB | 30.6% done | ETA(epoch): 0s + Train loss: 0.843176 (924.6s) ETA: 5832min + Val loss: 0.826712 [t_0.0-0.2=1.0790 t_0.2-0.4=0.9912 t_0.4-0.6=0.8639 t_0.6-0.8=0.7155 t_0.8-1.0=0.6508] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 153/499 (31% done) --- + [Epoch 153][10/40] loss=0.792883 avg=0.833491 VRAM=39.2GiB | 30.6% done | ETA(epoch): 693s + [Epoch 153][20/40] loss=0.755572 avg=0.842949 VRAM=39.0GiB | 30.7% done | ETA(epoch): 462s + [Epoch 153][30/40] loss=0.862441 avg=0.842709 VRAM=39.2GiB | 30.8% done | ETA(epoch): 231s + [Epoch 153][40/40] loss=0.800957 avg=0.845235 VRAM=39.0GiB | 30.8% done | ETA(epoch): 0s + Train loss: 0.845235 (924.5s) ETA: 5814min + Val loss: 0.824722 [t_0.0-0.2=1.0782 t_0.2-0.4=0.9994 t_0.4-0.6=0.8537 t_0.6-0.8=0.6955 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 154/499 (31% done) --- + [Epoch 154][10/40] loss=0.870305 avg=0.847662 VRAM=39.2GiB | 30.9% done | ETA(epoch): 693s + [Epoch 154][20/40] loss=0.902037 avg=0.853731 VRAM=39.0GiB | 30.9% done | ETA(epoch): 462s + [Epoch 154][30/40] loss=0.859222 avg=0.856638 VRAM=39.2GiB | 30.9% done | ETA(epoch): 231s + [Epoch 154][40/40] loss=0.976668 avg=0.858812 VRAM=39.0GiB | 31.0% done | ETA(epoch): 0s + Train loss: 0.858812 (924.6s) ETA: 5797min + Val loss: 0.869813 [t_0.0-0.2=1.0699 t_0.2-0.4=1.0118 t_0.4-0.6=0.8536 t_0.6-0.8=0.6952 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 155/499 (31% done) --- + [Epoch 155][10/40] loss=0.865103 avg=0.869571 VRAM=39.2GiB | 31.1% done | ETA(epoch): 693s + [Epoch 155][20/40] loss=0.865711 avg=0.860901 VRAM=39.0GiB | 31.1% done | ETA(epoch): 462s + [Epoch 155][30/40] loss=0.946299 avg=0.855169 VRAM=39.2GiB | 31.1% done | ETA(epoch): 231s + [Epoch 155][40/40] loss=0.813402 avg=0.853961 VRAM=39.0GiB | 31.2% done | ETA(epoch): 0s + Train loss: 0.853961 (924.5s) ETA: 5780min + Val loss: 0.851705 [t_0.0-0.2=1.0559 t_0.2-0.4=0.9908 t_0.4-0.6=0.8699 t_0.6-0.8=0.7121 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 156/499 (31% done) --- + [Epoch 156][10/40] loss=0.757427 avg=0.851415 VRAM=39.2GiB | 31.2% done | ETA(epoch): 693s + [Epoch 156][20/40] loss=0.836790 avg=0.835884 VRAM=39.0GiB | 31.3% done | ETA(epoch): 462s + [Epoch 156][30/40] loss=0.746374 avg=0.837643 VRAM=39.2GiB | 31.4% done | ETA(epoch): 231s + [Epoch 156][40/40] loss=0.905032 avg=0.842069 VRAM=39.0GiB | 31.4% done | ETA(epoch): 0s + Train loss: 0.842069 (924.8s) ETA: 5762min + Val loss: 0.844678 [t_0.0-0.2=1.0786 t_0.2-0.4=1.0155 t_0.4-0.6=0.8631 t_0.6-0.8=0.7004 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 157/499 (31% done) --- + [Epoch 157][10/40] loss=0.932658 avg=0.839160 VRAM=39.2GiB | 31.4% done | ETA(epoch): 693s + [Epoch 157][20/40] loss=0.881719 avg=0.846470 VRAM=39.0GiB | 31.5% done | ETA(epoch): 462s + [Epoch 157][30/40] loss=0.897202 avg=0.857084 VRAM=39.2GiB | 31.6% done | ETA(epoch): 231s + [Epoch 157][40/40] loss=0.830898 avg=0.851874 VRAM=39.0GiB | 31.6% done | ETA(epoch): 0s + Train loss: 0.851874 (925.0s) ETA: 5745min + Val loss: 0.839458 [t_0.0-0.2=1.0699 t_0.2-0.4=0.9983 t_0.4-0.6=0.8616 t_0.6-0.8=0.7173 t_0.8-1.0=0.6469] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 158/499 (32% done) --- + [Epoch 158][10/40] loss=0.923782 avg=0.881200 VRAM=39.2GiB | 31.6% done | ETA(epoch): 694s + [Epoch 158][20/40] loss=0.789277 avg=0.865110 VRAM=39.0GiB | 31.7% done | ETA(epoch): 462s + [Epoch 158][30/40] loss=0.791626 avg=0.863221 VRAM=39.2GiB | 31.8% done | ETA(epoch): 231s + [Epoch 158][40/40] loss=0.902564 avg=0.864601 VRAM=39.0GiB | 31.8% done | ETA(epoch): 0s + Train loss: 0.864601 (925.3s) ETA: 5727min + Val loss: 0.857335 [t_0.0-0.2=1.0720 t_0.2-0.4=1.0061 t_0.4-0.6=0.8125 t_0.6-0.8=0.7177 t_0.8-1.0=0.6316] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 159/499 (32% done) --- + [Epoch 159][10/40] loss=0.867058 avg=0.856936 VRAM=39.2GiB | 31.9% done | ETA(epoch): 695s + [Epoch 159][20/40] loss=0.907558 avg=0.869271 VRAM=39.0GiB | 31.9% done | ETA(epoch): 463s + [Epoch 159][30/40] loss=0.835620 avg=0.860185 VRAM=39.2GiB | 31.9% done | ETA(epoch): 232s + [Epoch 159][40/40] loss=0.910421 avg=0.859936 VRAM=39.0GiB | 32.0% done | ETA(epoch): 0s + Train loss: 0.859936 (926.5s) ETA: 5710min + Val loss: 0.840720 [t_0.0-0.2=1.0812 t_0.2-0.4=1.0202 t_0.4-0.6=0.8727 t_0.6-0.8=0.6765 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 160/499 (32% done) --- + [Epoch 160][10/40] loss=0.776236 avg=0.838861 VRAM=39.2GiB | 32.0% done | ETA(epoch): 694s + [Epoch 160][20/40] loss=0.920489 avg=0.842785 VRAM=39.0GiB | 32.1% done | ETA(epoch): 463s + [Epoch 160][30/40] loss=0.964290 avg=0.854657 VRAM=39.2GiB | 32.1% done | ETA(epoch): 231s + [Epoch 160][40/40] loss=0.724836 avg=0.853236 VRAM=39.0GiB | 32.2% done | ETA(epoch): 0s + Train loss: 0.853236 (925.7s) ETA: 5693min + Val loss: 0.864788 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0114 t_0.4-0.6=0.8267 t_0.6-0.8=0.7111 t_0.8-1.0=0.6489] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 161/499 (32% done) --- + [Epoch 161][10/40] loss=0.867517 avg=0.855745 VRAM=39.2GiB | 32.2% done | ETA(epoch): 694s + [Epoch 161][20/40] loss=0.881082 avg=0.866421 VRAM=39.0GiB | 32.3% done | ETA(epoch): 463s + [Epoch 161][30/40] loss=0.892287 avg=0.861643 VRAM=39.2GiB | 32.4% done | ETA(epoch): 231s + [Epoch 161][40/40] loss=0.784364 avg=0.857324 VRAM=39.0GiB | 32.4% done | ETA(epoch): 0s + Train loss: 0.857324 (925.3s) ETA: 5675min + Val loss: 0.857585 [t_0.0-0.2=1.0689 t_0.2-0.4=0.9933 t_0.4-0.6=0.8442 t_0.6-0.8=0.7404 t_0.8-1.0=0.6355] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 162/499 (32% done) --- + [Epoch 162][10/40] loss=0.737552 avg=0.841619 VRAM=39.2GiB | 32.5% done | ETA(epoch): 694s + [Epoch 162][20/40] loss=0.868240 avg=0.850062 VRAM=39.0GiB | 32.5% done | ETA(epoch): 462s + [Epoch 162][30/40] loss=0.843289 avg=0.852540 VRAM=39.2GiB | 32.6% done | ETA(epoch): 231s + [Epoch 162][40/40] loss=0.859007 avg=0.845904 VRAM=39.0GiB | 32.6% done | ETA(epoch): 0s + Train loss: 0.845904 (924.7s) ETA: 5658min + Val loss: 0.856892 [t_0.0-0.2=1.0658 t_0.2-0.4=0.9841 t_0.4-0.6=0.8487 t_0.6-0.8=0.7263 t_0.8-1.0=0.6282] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 163/499 (33% done) --- + [Epoch 163][10/40] loss=0.807316 avg=0.833729 VRAM=39.2GiB | 32.6% done | ETA(epoch): 693s + [Epoch 163][20/40] loss=0.871512 avg=0.848219 VRAM=39.0GiB | 32.7% done | ETA(epoch): 462s + [Epoch 163][30/40] loss=0.880071 avg=0.854349 VRAM=39.2GiB | 32.8% done | ETA(epoch): 231s + [Epoch 163][40/40] loss=0.814222 avg=0.854466 VRAM=39.0GiB | 32.8% done | ETA(epoch): 0s + Train loss: 0.854466 (924.2s) ETA: 5641min + Val loss: 0.924193 [t_0.0-0.2=1.0721 t_0.2-0.4=0.9944 t_0.4-0.6=0.8408 t_0.6-0.8=0.7244 t_0.8-1.0=0.6508] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 164/499 (33% done) --- + [Epoch 164][10/40] loss=0.845474 avg=0.863124 VRAM=39.2GiB | 32.9% done | ETA(epoch): 693s + [Epoch 164][20/40] loss=0.893954 avg=0.871379 VRAM=39.0GiB | 32.9% done | ETA(epoch): 462s + [Epoch 164][30/40] loss=0.786372 avg=0.865941 VRAM=39.2GiB | 33.0% done | ETA(epoch): 231s + [Epoch 164][40/40] loss=0.832449 avg=0.865203 VRAM=39.0GiB | 33.0% done | ETA(epoch): 0s + Train loss: 0.865203 (925.1s) ETA: 5623min + Val loss: 0.849085 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0171 t_0.4-0.6=0.8620 t_0.6-0.8=0.7026 t_0.8-1.0=0.6557] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 165/499 (33% done) --- + [Epoch 165][10/40] loss=0.900452 avg=0.885435 VRAM=39.2GiB | 33.1% done | ETA(epoch): 694s + [Epoch 165][20/40] loss=0.838355 avg=0.877300 VRAM=39.0GiB | 33.1% done | ETA(epoch): 463s + [Epoch 165][30/40] loss=0.933674 avg=0.876629 VRAM=39.2GiB | 33.1% done | ETA(epoch): 231s + [Epoch 165][40/40] loss=0.784265 avg=0.870303 VRAM=39.0GiB | 33.2% done | ETA(epoch): 0s + Train loss: 0.870303 (925.6s) ETA: 5606min + Val loss: 0.867115 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0073 t_0.4-0.6=0.8319 t_0.6-0.8=0.7270 t_0.8-1.0=0.6441] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 19.5/188.4 GiB (10.4%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 166/499 (33% done) --- + [Epoch 166][10/40] loss=0.899428 avg=0.853378 VRAM=39.2GiB | 33.2% done | ETA(epoch): 694s + [Epoch 166][20/40] loss=0.817210 avg=0.848701 VRAM=39.0GiB | 33.3% done | ETA(epoch): 463s + [Epoch 166][30/40] loss=0.792339 avg=0.838514 VRAM=39.2GiB | 33.4% done | ETA(epoch): 231s + [Epoch 166][40/40] loss=0.868474 avg=0.839923 VRAM=39.0GiB | 33.4% done | ETA(epoch): 0s + Train loss: 0.839923 (925.8s) ETA: 5589min + Val loss: 0.859417 [t_0.0-0.2=1.0611 t_0.2-0.4=0.9969 t_0.4-0.6=0.8282 t_0.6-0.8=0.7170 t_0.8-1.0=0.6514] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 167/499 (33% done) --- + [Epoch 167][10/40] loss=0.885122 avg=0.869383 VRAM=39.2GiB | 33.5% done | ETA(epoch): 694s + [Epoch 167][20/40] loss=0.822598 avg=0.862675 VRAM=39.0GiB | 33.5% done | ETA(epoch): 463s + [Epoch 167][30/40] loss=0.949664 avg=0.852654 VRAM=39.2GiB | 33.6% done | ETA(epoch): 231s + [Epoch 167][40/40] loss=0.852531 avg=0.859166 VRAM=39.0GiB | 33.6% done | ETA(epoch): 0s + Train loss: 0.859166 (925.6s) ETA: 5571min + Val loss: 0.860058 [t_0.0-0.2=1.0728 t_0.2-0.4=1.0142 t_0.4-0.6=0.8496 t_0.6-0.8=0.7060 t_0.8-1.0=0.6569] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 168/499 (34% done) --- + [Epoch 168][10/40] loss=0.933620 avg=0.890160 VRAM=39.2GiB | 33.7% done | ETA(epoch): 694s + [Epoch 168][20/40] loss=0.845112 avg=0.865753 VRAM=39.0GiB | 33.7% done | ETA(epoch): 463s + [Epoch 168][30/40] loss=0.917391 avg=0.863302 VRAM=39.2GiB | 33.8% done | ETA(epoch): 231s + [Epoch 168][40/40] loss=0.785573 avg=0.858889 VRAM=39.0GiB | 33.8% done | ETA(epoch): 0s + Train loss: 0.858889 (925.9s) ETA: 5554min + Val loss: 0.805322 [t_0.0-0.2=1.0493 t_0.2-0.4=0.9793 t_0.4-0.6=0.8547 t_0.6-0.8=0.7141 t_0.8-1.0=0.6341] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0168 (BEST) + Deleted old checkpoint: checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 169/499 (34% done) --- + [Epoch 169][10/40] loss=0.837176 avg=0.838005 VRAM=39.2GiB | 33.9% done | ETA(epoch): 694s + [Epoch 169][20/40] loss=0.847193 avg=0.859250 VRAM=39.0GiB | 33.9% done | ETA(epoch): 463s + [Epoch 169][30/40] loss=0.918369 avg=0.862363 VRAM=39.2GiB | 34.0% done | ETA(epoch): 231s + [Epoch 169][40/40] loss=0.875440 avg=0.867001 VRAM=39.0GiB | 34.0% done | ETA(epoch): 0s + Train loss: 0.867001 (925.8s) ETA: 5537min + Val loss: 0.836243 [t_0.0-0.2=1.0835 t_0.2-0.4=1.0073 t_0.4-0.6=0.8440 t_0.6-0.8=0.7163 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 170/499 (34% done) --- + [Epoch 170][10/40] loss=0.858210 avg=0.815176 VRAM=39.2GiB | 34.1% done | ETA(epoch): 694s + [Epoch 170][20/40] loss=0.773986 avg=0.829313 VRAM=39.0GiB | 34.1% done | ETA(epoch): 463s + [Epoch 170][30/40] loss=0.951393 avg=0.842685 VRAM=39.2GiB | 34.2% done | ETA(epoch): 231s + [Epoch 170][40/40] loss=0.939695 avg=0.846293 VRAM=39.0GiB | 34.2% done | ETA(epoch): 0s + Train loss: 0.846293 (925.8s) ETA: 5520min + Val loss: 0.841722 [t_0.0-0.2=1.0867 t_0.2-0.4=1.0021 t_0.4-0.6=0.8578 t_0.6-0.8=0.7012 t_0.8-1.0=0.6442] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 171/499 (34% done) --- + [Epoch 171][10/40] loss=0.717236 avg=0.834684 VRAM=39.2GiB | 34.2% done | ETA(epoch): 694s + [Epoch 171][20/40] loss=0.873664 avg=0.838696 VRAM=39.0GiB | 34.3% done | ETA(epoch): 462s + [Epoch 171][30/40] loss=0.906145 avg=0.838919 VRAM=39.2GiB | 34.4% done | ETA(epoch): 231s + [Epoch 171][40/40] loss=0.786692 avg=0.846364 VRAM=39.0GiB | 34.4% done | ETA(epoch): 0s + Train loss: 0.846364 (924.4s) ETA: 5502min + Val loss: 0.850464 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9909 t_0.4-0.6=0.8465 t_0.6-0.8=0.7100 t_0.8-1.0=0.6263] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0171 +[MEM @ epoch 171 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 172/499 (34% done) --- + [Epoch 172][10/40] loss=0.894157 avg=0.847810 VRAM=39.2GiB | 34.4% done | ETA(epoch): 693s + [Epoch 172][20/40] loss=0.873271 avg=0.865916 VRAM=39.0GiB | 34.5% done | ETA(epoch): 462s + [Epoch 172][30/40] loss=0.895881 avg=0.862694 VRAM=39.2GiB | 34.5% done | ETA(epoch): 231s + [Epoch 172][40/40] loss=0.741002 avg=0.853083 VRAM=39.0GiB | 34.6% done | ETA(epoch): 0s + Train loss: 0.853083 (924.1s) ETA: 5485min + Val loss: 0.840876 [t_0.0-0.2=1.0796 t_0.2-0.4=0.9845 t_0.4-0.6=0.8572 t_0.6-0.8=0.7240 t_0.8-1.0=0.6520] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 173/499 (35% done) --- + [Epoch 173][10/40] loss=0.996519 avg=0.872485 VRAM=39.2GiB | 34.6% done | ETA(epoch): 693s + [Epoch 173][20/40] loss=0.801341 avg=0.865108 VRAM=39.0GiB | 34.7% done | ETA(epoch): 462s + [Epoch 173][30/40] loss=0.714645 avg=0.862116 VRAM=39.2GiB | 34.8% done | ETA(epoch): 231s + [Epoch 173][40/40] loss=0.905940 avg=0.857560 VRAM=39.0GiB | 34.8% done | ETA(epoch): 0s + Train loss: 0.857560 (925.0s) ETA: 5468min + Val loss: 0.861101 [t_0.0-0.2=1.0733 t_0.2-0.4=1.0131 t_0.4-0.6=0.8925 t_0.6-0.8=0.7022 t_0.8-1.0=0.6372] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 174/499 (35% done) --- + [Epoch 174][10/40] loss=0.868581 avg=0.831081 VRAM=39.2GiB | 34.8% done | ETA(epoch): 694s + [Epoch 174][20/40] loss=0.812329 avg=0.831691 VRAM=39.0GiB | 34.9% done | ETA(epoch): 462s + [Epoch 174][30/40] loss=0.805258 avg=0.848413 VRAM=39.2GiB | 34.9% done | ETA(epoch): 231s + [Epoch 174][40/40] loss=0.819655 avg=0.846686 VRAM=39.0GiB | 35.0% done | ETA(epoch): 0s + Train loss: 0.846686 (925.0s) ETA: 5450min + Val loss: 0.865819 [t_0.0-0.2=1.0721 t_0.2-0.4=1.0151 t_0.4-0.6=0.8345 t_0.6-0.8=0.7103 t_0.8-1.0=0.6399] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 175/499 (35% done) --- + [Epoch 175][10/40] loss=0.834481 avg=0.833736 VRAM=39.2GiB | 35.0% done | ETA(epoch): 694s + [Epoch 175][20/40] loss=0.903217 avg=0.843797 VRAM=39.0GiB | 35.1% done | ETA(epoch): 462s + [Epoch 175][30/40] loss=0.782750 avg=0.846478 VRAM=39.2GiB | 35.1% done | ETA(epoch): 231s + [Epoch 175][40/40] loss=0.765922 avg=0.849411 VRAM=39.0GiB | 35.2% done | ETA(epoch): 0s + Train loss: 0.849411 (925.0s) ETA: 5433min + Val loss: 0.894628 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9930 t_0.4-0.6=0.8749 t_0.6-0.8=0.7395 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 176/499 (35% done) --- + [Epoch 176][10/40] loss=0.886103 avg=0.873977 VRAM=39.2GiB | 35.2% done | ETA(epoch): 693s + [Epoch 176][20/40] loss=0.845460 avg=0.860602 VRAM=39.0GiB | 35.3% done | ETA(epoch): 462s + [Epoch 176][30/40] loss=0.873193 avg=0.869522 VRAM=39.2GiB | 35.4% done | ETA(epoch): 231s + [Epoch 176][40/40] loss=0.817442 avg=0.865314 VRAM=39.0GiB | 35.4% done | ETA(epoch): 0s + Train loss: 0.865314 (924.7s) ETA: 5416min + Val loss: 0.863791 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9995 t_0.4-0.6=0.8461 t_0.6-0.8=0.7014 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 177/499 (35% done) --- + [Epoch 177][10/40] loss=0.885105 avg=0.871441 VRAM=39.2GiB | 35.4% done | ETA(epoch): 694s + [Epoch 177][20/40] loss=0.949096 avg=0.869627 VRAM=39.0GiB | 35.5% done | ETA(epoch): 463s + [Epoch 177][30/40] loss=0.913680 avg=0.870382 VRAM=39.2GiB | 35.5% done | ETA(epoch): 231s + [Epoch 177][40/40] loss=0.796236 avg=0.867032 VRAM=39.0GiB | 35.6% done | ETA(epoch): 0s + Train loss: 0.867032 (925.1s) ETA: 5399min + Val loss: 0.881285 [t_0.0-0.2=1.0738 t_0.2-0.4=1.0096 t_0.4-0.6=0.8339 t_0.6-0.8=0.7407 t_0.8-1.0=0.6527] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 178/499 (36% done) --- + [Epoch 178][10/40] loss=0.752067 avg=0.845419 VRAM=39.2GiB | 35.6% done | ETA(epoch): 693s + [Epoch 178][20/40] loss=0.882965 avg=0.852257 VRAM=39.0GiB | 35.7% done | ETA(epoch): 462s + [Epoch 178][30/40] loss=0.956967 avg=0.856668 VRAM=39.2GiB | 35.8% done | ETA(epoch): 231s + [Epoch 178][40/40] loss=0.905651 avg=0.854915 VRAM=39.0GiB | 35.8% done | ETA(epoch): 0s + Train loss: 0.854915 (924.5s) ETA: 5382min + Val loss: 0.830949 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0108 t_0.4-0.6=0.8192 t_0.6-0.8=0.7171 t_0.8-1.0=0.6343] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 179/499 (36% done) --- + [Epoch 179][10/40] loss=0.876182 avg=0.874845 VRAM=39.2GiB | 35.9% done | ETA(epoch): 692s + [Epoch 179][20/40] loss=0.950937 avg=0.871403 VRAM=39.0GiB | 35.9% done | ETA(epoch): 462s + [Epoch 179][30/40] loss=0.833761 avg=0.856994 VRAM=39.2GiB | 35.9% done | ETA(epoch): 231s + [Epoch 179][40/40] loss=0.915666 avg=0.850939 VRAM=39.0GiB | 36.0% done | ETA(epoch): 0s + Train loss: 0.850939 (923.8s) ETA: 5364min + Val loss: 0.844039 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9803 t_0.4-0.6=0.8129 t_0.6-0.8=0.7118 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 180/499 (36% done) --- + [Epoch 180][10/40] loss=0.925749 avg=0.835608 VRAM=39.2GiB | 36.0% done | ETA(epoch): 693s + [Epoch 180][20/40] loss=0.744544 avg=0.854254 VRAM=39.0GiB | 36.1% done | ETA(epoch): 462s + [Epoch 180][30/40] loss=0.731505 avg=0.860430 VRAM=39.2GiB | 36.1% done | ETA(epoch): 231s + [Epoch 180][40/40] loss=0.741883 avg=0.859236 VRAM=39.0GiB | 36.2% done | ETA(epoch): 0s + Train loss: 0.859236 (924.2s) ETA: 5347min + Val loss: 0.841503 [t_0.0-0.2=1.0656 t_0.2-0.4=0.9899 t_0.4-0.6=0.8632 t_0.6-0.8=0.7053 t_0.8-1.0=0.6463] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 181/499 (36% done) --- + [Epoch 181][10/40] loss=0.879434 avg=0.888822 VRAM=39.2GiB | 36.2% done | ETA(epoch): 693s + [Epoch 181][20/40] loss=0.783406 avg=0.874287 VRAM=39.0GiB | 36.3% done | ETA(epoch): 462s + [Epoch 181][30/40] loss=0.919939 avg=0.867534 VRAM=39.2GiB | 36.4% done | ETA(epoch): 231s + [Epoch 181][40/40] loss=0.852017 avg=0.874955 VRAM=39.0GiB | 36.4% done | ETA(epoch): 0s + Train loss: 0.874955 (924.3s) ETA: 5330min + Val loss: 0.837734 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9890 t_0.4-0.6=0.8129 t_0.6-0.8=0.7066 t_0.8-1.0=0.6275] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 182/499 (36% done) --- + [Epoch 182][10/40] loss=0.857443 avg=0.857880 VRAM=39.2GiB | 36.4% done | ETA(epoch): 693s + [Epoch 182][20/40] loss=0.762737 avg=0.866504 VRAM=39.0GiB | 36.5% done | ETA(epoch): 462s + [Epoch 182][30/40] loss=0.838741 avg=0.860805 VRAM=39.2GiB | 36.5% done | ETA(epoch): 231s + [Epoch 182][40/40] loss=0.930648 avg=0.866701 VRAM=39.0GiB | 36.6% done | ETA(epoch): 0s + Train loss: 0.866701 (924.2s) ETA: 5313min + Val loss: 0.847882 [t_0.0-0.2=1.0667 t_0.2-0.4=1.0079 t_0.4-0.6=0.8906 t_0.6-0.8=0.7171 t_0.8-1.0=0.6363] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 183/499 (37% done) --- + [Epoch 183][10/40] loss=0.798048 avg=0.861415 VRAM=39.2GiB | 36.6% done | ETA(epoch): 693s + [Epoch 183][20/40] loss=0.941100 avg=0.856294 VRAM=39.0GiB | 36.7% done | ETA(epoch): 462s + [Epoch 183][30/40] loss=0.881804 avg=0.856178 VRAM=39.2GiB | 36.8% done | ETA(epoch): 231s + [Epoch 183][40/40] loss=0.814429 avg=0.848935 VRAM=39.0GiB | 36.8% done | ETA(epoch): 0s + Train loss: 0.848935 (924.2s) ETA: 5295min + Val loss: 0.864985 [t_0.0-0.2=1.0719 t_0.2-0.4=0.9983 t_0.4-0.6=0.8534 t_0.6-0.8=0.7024 t_0.8-1.0=0.6312] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 184/499 (37% done) --- + [Epoch 184][10/40] loss=0.817641 avg=0.862250 VRAM=39.2GiB | 36.9% done | ETA(epoch): 693s + [Epoch 184][20/40] loss=0.846929 avg=0.847177 VRAM=39.0GiB | 36.9% done | ETA(epoch): 462s + [Epoch 184][30/40] loss=0.811216 avg=0.840024 VRAM=39.2GiB | 37.0% done | ETA(epoch): 231s + [Epoch 184][40/40] loss=0.783048 avg=0.842157 VRAM=39.0GiB | 37.0% done | ETA(epoch): 0s + Train loss: 0.842157 (924.1s) ETA: 5278min + Val loss: 0.829388 [t_0.0-0.2=1.0633 t_0.2-0.4=0.9921 t_0.4-0.6=0.8240 t_0.6-0.8=0.7326 t_0.8-1.0=0.6581] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 185/499 (37% done) --- + [Epoch 185][10/40] loss=0.896955 avg=0.826528 VRAM=39.2GiB | 37.0% done | ETA(epoch): 693s + [Epoch 185][20/40] loss=0.898036 avg=0.861674 VRAM=39.0GiB | 37.1% done | ETA(epoch): 462s + [Epoch 185][30/40] loss=0.772430 avg=0.867413 VRAM=39.2GiB | 37.1% done | ETA(epoch): 231s + [Epoch 185][40/40] loss=0.930045 avg=0.866748 VRAM=39.0GiB | 37.2% done | ETA(epoch): 0s + Train loss: 0.866748 (923.9s) ETA: 5261min + Val loss: 0.868175 [t_0.0-0.2=1.0717 t_0.2-0.4=0.9942 t_0.4-0.6=0.8349 t_0.6-0.8=0.7068 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 186/499 (37% done) --- + [Epoch 186][10/40] loss=0.867263 avg=0.854949 VRAM=39.2GiB | 37.2% done | ETA(epoch): 692s + [Epoch 186][20/40] loss=0.880216 avg=0.862808 VRAM=39.0GiB | 37.3% done | ETA(epoch): 462s + [Epoch 186][30/40] loss=0.852965 avg=0.864410 VRAM=39.2GiB | 37.4% done | ETA(epoch): 231s + [Epoch 186][40/40] loss=0.856054 avg=0.862433 VRAM=39.0GiB | 37.4% done | ETA(epoch): 0s + Train loss: 0.862433 (923.6s) ETA: 5244min + Val loss: 0.871487 [t_0.0-0.2=1.0693 t_0.2-0.4=0.9908 t_0.4-0.6=0.8617 t_0.6-0.8=0.7260 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 187/499 (37% done) --- + [Epoch 187][10/40] loss=0.841447 avg=0.843654 VRAM=39.2GiB | 37.5% done | ETA(epoch): 692s + [Epoch 187][20/40] loss=0.817301 avg=0.852325 VRAM=39.0GiB | 37.5% done | ETA(epoch): 462s + [Epoch 187][30/40] loss=0.793660 avg=0.856110 VRAM=39.2GiB | 37.5% done | ETA(epoch): 231s + [Epoch 187][40/40] loss=0.965020 avg=0.853229 VRAM=39.0GiB | 37.6% done | ETA(epoch): 0s + Train loss: 0.853229 (923.6s) ETA: 5227min + Val loss: 0.845068 [t_0.0-0.2=1.0626 t_0.2-0.4=1.0196 t_0.4-0.6=0.8765 t_0.6-0.8=0.7315 t_0.8-1.0=0.6257] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 188/499 (38% done) --- + [Epoch 188][10/40] loss=0.898346 avg=0.868863 VRAM=39.2GiB | 37.6% done | ETA(epoch): 692s + [Epoch 188][20/40] loss=0.853930 avg=0.871780 VRAM=39.0GiB | 37.7% done | ETA(epoch): 462s + [Epoch 188][30/40] loss=0.767251 avg=0.867595 VRAM=39.2GiB | 37.8% done | ETA(epoch): 231s + [Epoch 188][40/40] loss=0.858916 avg=0.864074 VRAM=39.0GiB | 37.8% done | ETA(epoch): 0s + Train loss: 0.864074 (923.5s) ETA: 5209min + Val loss: 0.829002 [t_0.0-0.2=1.0590 t_0.2-0.4=0.9749 t_0.4-0.6=0.8394 t_0.6-0.8=0.7348 t_0.8-1.0=0.6313] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 189/499 (38% done) --- + [Epoch 189][10/40] loss=0.762533 avg=0.844594 VRAM=39.2GiB | 37.9% done | ETA(epoch): 693s + [Epoch 189][20/40] loss=0.830586 avg=0.857098 VRAM=39.0GiB | 37.9% done | ETA(epoch): 462s + [Epoch 189][30/40] loss=0.879874 avg=0.851109 VRAM=39.2GiB | 38.0% done | ETA(epoch): 231s + [Epoch 189][40/40] loss=0.774465 avg=0.855333 VRAM=39.0GiB | 38.0% done | ETA(epoch): 0s + Train loss: 0.855333 (923.9s) ETA: 5192min + Val loss: 0.804852 [t_0.0-0.2=1.0821 t_0.2-0.4=1.0229 t_0.4-0.6=0.8245 t_0.6-0.8=0.6974 t_0.8-1.0=0.6435] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0189 (BEST) + Deleted old checkpoint: checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 190/499 (38% done) --- + [Epoch 190][10/40] loss=0.785228 avg=0.824716 VRAM=39.2GiB | 38.0% done | ETA(epoch): 693s + [Epoch 190][20/40] loss=0.884708 avg=0.851666 VRAM=39.0GiB | 38.1% done | ETA(epoch): 462s + [Epoch 190][30/40] loss=0.884409 avg=0.853505 VRAM=39.2GiB | 38.1% done | ETA(epoch): 231s + [Epoch 190][40/40] loss=0.832704 avg=0.861335 VRAM=39.0GiB | 38.2% done | ETA(epoch): 0s + Train loss: 0.861335 (924.2s) ETA: 5175min + Val loss: 0.847867 [t_0.0-0.2=1.0815 t_0.2-0.4=0.9756 t_0.4-0.6=0.8714 t_0.6-0.8=0.7226 t_0.8-1.0=0.6376] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 191/499 (38% done) --- + [Epoch 191][10/40] loss=0.815170 avg=0.842372 VRAM=39.2GiB | 38.2% done | ETA(epoch): 693s + [Epoch 191][20/40] loss=0.834984 avg=0.855154 VRAM=39.0GiB | 38.3% done | ETA(epoch): 462s + [Epoch 191][30/40] loss=0.862816 avg=0.849479 VRAM=39.2GiB | 38.4% done | ETA(epoch): 231s + [Epoch 191][40/40] loss=0.865171 avg=0.850522 VRAM=39.0GiB | 38.4% done | ETA(epoch): 0s + Train loss: 0.850522 (924.0s) ETA: 5158min + Val loss: 0.822483 [t_0.0-0.2=1.0681 t_0.2-0.4=1.0194 t_0.4-0.6=0.8032 t_0.6-0.8=0.7204 t_0.8-1.0=0.6401] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0191 + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 192/499 (38% done) --- + [Epoch 192][10/40] loss=0.907253 avg=0.851316 VRAM=39.2GiB | 38.5% done | ETA(epoch): 693s + [Epoch 192][20/40] loss=0.768025 avg=0.841446 VRAM=39.0GiB | 38.5% done | ETA(epoch): 461s + [Epoch 192][30/40] loss=0.895643 avg=0.847172 VRAM=39.2GiB | 38.6% done | ETA(epoch): 231s + [Epoch 192][40/40] loss=0.949218 avg=0.857649 VRAM=39.0GiB | 38.6% done | ETA(epoch): 0s + Train loss: 0.857649 (923.4s) ETA: 5141min + Val loss: 0.826543 [t_0.0-0.2=1.0695 t_0.2-0.4=0.9734 t_0.4-0.6=0.8139 t_0.6-0.8=0.7129 t_0.8-1.0=0.6471] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0192 +[MEM @ epoch 192 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 193/499 (39% done) --- + [Epoch 193][10/40] loss=0.981719 avg=0.859243 VRAM=39.2GiB | 38.6% done | ETA(epoch): 692s + [Epoch 193][20/40] loss=0.934046 avg=0.854605 VRAM=39.0GiB | 38.7% done | ETA(epoch): 462s + [Epoch 193][30/40] loss=0.722640 avg=0.844648 VRAM=39.2GiB | 38.8% done | ETA(epoch): 231s + [Epoch 193][40/40] loss=0.818742 avg=0.850648 VRAM=39.0GiB | 38.8% done | ETA(epoch): 0s + Train loss: 0.850648 (924.2s) ETA: 5124min + Val loss: 0.861364 [t_0.0-0.2=1.0650 t_0.2-0.4=1.0282 t_0.4-0.6=0.8511 t_0.6-0.8=0.6948 t_0.8-1.0=0.6409] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 194/499 (39% done) --- + [Epoch 194][10/40] loss=0.834557 avg=0.854833 VRAM=39.2GiB | 38.9% done | ETA(epoch): 693s + [Epoch 194][20/40] loss=0.823324 avg=0.839758 VRAM=39.0GiB | 38.9% done | ETA(epoch): 462s + [Epoch 194][30/40] loss=0.880882 avg=0.855215 VRAM=39.2GiB | 39.0% done | ETA(epoch): 231s + [Epoch 194][40/40] loss=0.871511 avg=0.855835 VRAM=39.0GiB | 39.0% done | ETA(epoch): 0s + Train loss: 0.855835 (924.2s) ETA: 5107min + Val loss: 0.926405 [t_0.0-0.2=1.0783 t_0.2-0.4=1.0087 t_0.4-0.6=0.8182 t_0.6-0.8=0.7137 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 195/499 (39% done) --- + [Epoch 195][10/40] loss=0.942420 avg=0.839657 VRAM=39.2GiB | 39.1% done | ETA(epoch): 693s + [Epoch 195][20/40] loss=0.888229 avg=0.850661 VRAM=39.0GiB | 39.1% done | ETA(epoch): 462s + [Epoch 195][30/40] loss=0.756701 avg=0.840158 VRAM=39.2GiB | 39.1% done | ETA(epoch): 231s + [Epoch 195][40/40] loss=0.763299 avg=0.847650 VRAM=39.0GiB | 39.2% done | ETA(epoch): 0s + Train loss: 0.847650 (924.7s) ETA: 5090min + Val loss: 0.870606 [t_0.0-0.2=1.0700 t_0.2-0.4=1.0072 t_0.4-0.6=0.8255 t_0.6-0.8=0.7258 t_0.8-1.0=0.6596] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 196/499 (39% done) --- + [Epoch 196][10/40] loss=0.929637 avg=0.843196 VRAM=39.2GiB | 39.2% done | ETA(epoch): 692s + [Epoch 196][20/40] loss=0.764671 avg=0.846408 VRAM=39.0GiB | 39.3% done | ETA(epoch): 462s + [Epoch 196][30/40] loss=0.867335 avg=0.847714 VRAM=39.2GiB | 39.4% done | ETA(epoch): 231s + [Epoch 196][40/40] loss=0.930352 avg=0.841857 VRAM=39.0GiB | 39.4% done | ETA(epoch): 0s + Train loss: 0.841857 (924.3s) ETA: 5072min + Val loss: 0.857461 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9774 t_0.4-0.6=0.8358 t_0.6-0.8=0.7106 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 197/499 (39% done) --- + [Epoch 197][10/40] loss=0.945163 avg=0.884434 VRAM=39.2GiB | 39.5% done | ETA(epoch): 693s + [Epoch 197][20/40] loss=0.921477 avg=0.890972 VRAM=39.0GiB | 39.5% done | ETA(epoch): 462s + [Epoch 197][30/40] loss=0.771897 avg=0.868766 VRAM=39.2GiB | 39.6% done | ETA(epoch): 231s + [Epoch 197][40/40] loss=0.820739 avg=0.866650 VRAM=39.0GiB | 39.6% done | ETA(epoch): 0s + Train loss: 0.866650 (924.3s) ETA: 5055min + Val loss: 0.891112 [t_0.0-0.2=1.0754 t_0.2-0.4=0.9979 t_0.4-0.6=0.8502 t_0.6-0.8=0.7127 t_0.8-1.0=0.6530] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 198/499 (40% done) --- + [Epoch 198][10/40] loss=0.867272 avg=0.873501 VRAM=39.2GiB | 39.6% done | ETA(epoch): 693s + [Epoch 198][20/40] loss=0.857214 avg=0.874260 VRAM=39.0GiB | 39.7% done | ETA(epoch): 462s + [Epoch 198][30/40] loss=0.785449 avg=0.870833 VRAM=39.2GiB | 39.8% done | ETA(epoch): 231s + [Epoch 198][40/40] loss=0.899931 avg=0.872506 VRAM=39.0GiB | 39.8% done | ETA(epoch): 0s + Train loss: 0.872506 (923.6s) ETA: 5038min + Val loss: 0.880088 [t_0.0-0.2=1.0822 t_0.2-0.4=1.0077 t_0.4-0.6=0.8501 t_0.6-0.8=0.7086 t_0.8-1.0=0.6522] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 199/499 (40% done) --- + [Epoch 199][10/40] loss=0.922804 avg=0.859570 VRAM=39.2GiB | 39.9% done | ETA(epoch): 693s + [Epoch 199][20/40] loss=0.902959 avg=0.865898 VRAM=39.0GiB | 39.9% done | ETA(epoch): 462s + [Epoch 199][30/40] loss=0.906421 avg=0.865684 VRAM=39.2GiB | 40.0% done | ETA(epoch): 231s + [Epoch 199][40/40] loss=0.830670 avg=0.858469 VRAM=39.0GiB | 40.0% done | ETA(epoch): 0s + Train loss: 0.858469 (924.0s) ETA: 5021min + Val loss: 0.843337 [t_0.0-0.2=1.0679 t_0.2-0.4=0.9973 t_0.4-0.6=0.8235 t_0.6-0.8=0.6934 t_0.8-1.0=0.6395] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 200/499 (40% done) --- + [MilestoneVis] train_0 step 8000 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 8000 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 8000 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 8000 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 8000 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 8000 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 8000 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 8000 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 8000 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 8000 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 200][10/40] loss=0.954189 avg=0.865147 VRAM=39.2GiB | 40.1% done | ETA(epoch): 2953s + [Epoch 200][20/40] loss=0.843542 avg=0.870354 VRAM=39.0GiB | 40.1% done | ETA(epoch): 1215s + [Epoch 200][30/40] loss=0.941894 avg=0.869576 VRAM=39.2GiB | 40.2% done | ETA(epoch): 482s + [Epoch 200][40/40] loss=0.756949 avg=0.868833 VRAM=39.0GiB | 40.2% done | ETA(epoch): 0s + Train loss: 0.868833 (1677.0s) ETA: 5023min + Val loss: 0.856572 [t_0.0-0.2=1.0813 t_0.2-0.4=0.9872 t_0.4-0.6=0.8470 t_0.6-0.8=0.7271 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 201/499 (40% done) --- + [Epoch 201][10/40] loss=0.893062 avg=0.868565 VRAM=39.2GiB | 40.2% done | ETA(epoch): 692s + [Epoch 201][20/40] loss=0.833928 avg=0.845344 VRAM=39.0GiB | 40.3% done | ETA(epoch): 462s + [Epoch 201][30/40] loss=0.733149 avg=0.833824 VRAM=39.2GiB | 40.4% done | ETA(epoch): 231s + [Epoch 201][40/40] loss=0.781707 avg=0.832201 VRAM=39.0GiB | 40.4% done | ETA(epoch): 0s + Train loss: 0.832201 (924.0s) ETA: 5005min + Val loss: 0.851958 [t_0.0-0.2=1.0654 t_0.2-0.4=1.0395 t_0.4-0.6=0.8345 t_0.6-0.8=0.6995 t_0.8-1.0=0.6287] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 202/499 (40% done) --- + [Epoch 202][10/40] loss=0.746815 avg=0.866478 VRAM=39.2GiB | 40.5% done | ETA(epoch): 693s + [Epoch 202][20/40] loss=0.846605 avg=0.871248 VRAM=39.0GiB | 40.5% done | ETA(epoch): 462s + [Epoch 202][30/40] loss=0.844718 avg=0.867272 VRAM=39.2GiB | 40.6% done | ETA(epoch): 231s + [Epoch 202][40/40] loss=0.799451 avg=0.854457 VRAM=39.0GiB | 40.6% done | ETA(epoch): 0s + Train loss: 0.854457 (924.2s) ETA: 4988min + Val loss: 0.826306 [t_0.0-0.2=1.0539 t_0.2-0.4=1.0069 t_0.4-0.6=0.8596 t_0.6-0.8=0.7232 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 203/499 (41% done) --- + [Epoch 203][10/40] loss=0.860971 avg=0.848500 VRAM=39.2GiB | 40.6% done | ETA(epoch): 693s + [Epoch 203][20/40] loss=0.826159 avg=0.840689 VRAM=39.0GiB | 40.7% done | ETA(epoch): 462s + [Epoch 203][30/40] loss=0.970854 avg=0.849218 VRAM=39.2GiB | 40.8% done | ETA(epoch): 231s + [Epoch 203][40/40] loss=0.925361 avg=0.856679 VRAM=39.0GiB | 40.8% done | ETA(epoch): 0s + Train loss: 0.856679 (924.0s) ETA: 4971min + Val loss: 0.861942 [t_0.0-0.2=1.0790 t_0.2-0.4=1.0332 t_0.4-0.6=0.8305 t_0.6-0.8=0.6874 t_0.8-1.0=0.6316] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 204/499 (41% done) --- + [Epoch 204][10/40] loss=0.830022 avg=0.837025 VRAM=39.2GiB | 40.8% done | ETA(epoch): 693s + [Epoch 204][20/40] loss=0.871024 avg=0.852156 VRAM=39.0GiB | 40.9% done | ETA(epoch): 462s + [Epoch 204][30/40] loss=0.816128 avg=0.845785 VRAM=39.2GiB | 40.9% done | ETA(epoch): 231s + [Epoch 204][40/40] loss=0.872412 avg=0.845465 VRAM=39.0GiB | 41.0% done | ETA(epoch): 0s + Train loss: 0.845465 (924.1s) ETA: 4954min + Val loss: 0.859430 [t_0.0-0.2=1.0553 t_0.2-0.4=0.9970 t_0.4-0.6=0.8195 t_0.6-0.8=0.7091 t_0.8-1.0=0.6261] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 205/499 (41% done) --- + [Epoch 205][10/40] loss=0.844622 avg=0.864753 VRAM=39.2GiB | 41.0% done | ETA(epoch): 693s + [Epoch 205][20/40] loss=0.923006 avg=0.847740 VRAM=39.0GiB | 41.1% done | ETA(epoch): 462s + [Epoch 205][30/40] loss=0.693031 avg=0.853439 VRAM=39.2GiB | 41.1% done | ETA(epoch): 231s + [Epoch 205][40/40] loss=0.800898 avg=0.849664 VRAM=39.0GiB | 41.2% done | ETA(epoch): 0s + Train loss: 0.849664 (923.9s) ETA: 4937min + Val loss: 0.863297 [t_0.0-0.2=1.0768 t_0.2-0.4=0.9876 t_0.4-0.6=0.8779 t_0.6-0.8=0.7201 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 206/499 (41% done) --- + [Epoch 206][10/40] loss=0.827618 avg=0.850753 VRAM=39.2GiB | 41.2% done | ETA(epoch): 693s + [Epoch 206][20/40] loss=0.840965 avg=0.862163 VRAM=39.0GiB | 41.3% done | ETA(epoch): 462s + [Epoch 206][30/40] loss=0.811782 avg=0.866723 VRAM=39.2GiB | 41.3% done | ETA(epoch): 231s + [Epoch 206][40/40] loss=0.964847 avg=0.863319 VRAM=39.0GiB | 41.4% done | ETA(epoch): 0s + Train loss: 0.863319 (923.8s) ETA: 4919min + Val loss: 0.836331 [t_0.0-0.2=1.0651 t_0.2-0.4=0.9861 t_0.4-0.6=0.8283 t_0.6-0.8=0.7222 t_0.8-1.0=0.6302] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 207/499 (41% done) --- + [Epoch 207][10/40] loss=0.795520 avg=0.815033 VRAM=39.2GiB | 41.4% done | ETA(epoch): 693s + [Epoch 207][20/40] loss=0.868197 avg=0.819141 VRAM=39.0GiB | 41.5% done | ETA(epoch): 462s + [Epoch 207][30/40] loss=0.967465 avg=0.839597 VRAM=39.2GiB | 41.5% done | ETA(epoch): 231s + [Epoch 207][40/40] loss=0.757397 avg=0.838486 VRAM=39.0GiB | 41.6% done | ETA(epoch): 0s + Train loss: 0.838486 (923.8s) ETA: 4902min + Val loss: 0.859650 [t_0.0-0.2=1.0570 t_0.2-0.4=0.9910 t_0.4-0.6=0.8733 t_0.6-0.8=0.7350 t_0.8-1.0=0.6318] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 208/499 (42% done) --- + [Epoch 208][10/40] loss=0.903786 avg=0.878903 VRAM=39.2GiB | 41.6% done | ETA(epoch): 693s + [Epoch 208][20/40] loss=0.833094 avg=0.854881 VRAM=39.0GiB | 41.7% done | ETA(epoch): 462s + [Epoch 208][30/40] loss=0.953072 avg=0.850675 VRAM=39.2GiB | 41.8% done | ETA(epoch): 231s + [Epoch 208][40/40] loss=0.771734 avg=0.845554 VRAM=39.0GiB | 41.8% done | ETA(epoch): 0s + Train loss: 0.845554 (923.5s) ETA: 4885min + Val loss: 0.855111 [t_0.0-0.2=1.0848 t_0.2-0.4=0.9844 t_0.4-0.6=0.8919 t_0.6-0.8=0.6827 t_0.8-1.0=0.6471] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 209/499 (42% done) --- + [Epoch 209][10/40] loss=0.877219 avg=0.873750 VRAM=39.2GiB | 41.9% done | ETA(epoch): 693s + [Epoch 209][20/40] loss=0.842999 avg=0.869218 VRAM=39.0GiB | 41.9% done | ETA(epoch): 462s + [Epoch 209][30/40] loss=0.819508 avg=0.862712 VRAM=39.2GiB | 41.9% done | ETA(epoch): 231s + [Epoch 209][40/40] loss=0.822714 avg=0.857444 VRAM=39.0GiB | 42.0% done | ETA(epoch): 0s + Train loss: 0.857444 (924.4s) ETA: 4868min + Val loss: 0.845139 [t_0.0-0.2=1.0763 t_0.2-0.4=0.9758 t_0.4-0.6=0.8727 t_0.6-0.8=0.7200 t_0.8-1.0=0.6369] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 210/499 (42% done) --- + [Epoch 210][10/40] loss=0.944240 avg=0.835871 VRAM=39.2GiB | 42.0% done | ETA(epoch): 693s + [Epoch 210][20/40] loss=0.831304 avg=0.860419 VRAM=39.0GiB | 42.1% done | ETA(epoch): 462s + [Epoch 210][30/40] loss=0.837249 avg=0.854743 VRAM=39.2GiB | 42.1% done | ETA(epoch): 231s + [Epoch 210][40/40] loss=0.896602 avg=0.855846 VRAM=39.0GiB | 42.2% done | ETA(epoch): 0s + Train loss: 0.855846 (924.1s) ETA: 4851min + Val loss: 0.834539 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9762 t_0.4-0.6=0.8452 t_0.6-0.8=0.7166 t_0.8-1.0=0.6489] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 211/499 (42% done) --- + [Epoch 211][10/40] loss=0.892694 avg=0.847341 VRAM=39.2GiB | 42.2% done | ETA(epoch): 693s + [Epoch 211][20/40] loss=0.919716 avg=0.844478 VRAM=39.0GiB | 42.3% done | ETA(epoch): 462s + [Epoch 211][30/40] loss=0.886041 avg=0.846519 VRAM=39.2GiB | 42.4% done | ETA(epoch): 231s + [Epoch 211][40/40] loss=0.901244 avg=0.843230 VRAM=39.0GiB | 42.4% done | ETA(epoch): 0s + Train loss: 0.843230 (924.1s) ETA: 4833min + Val loss: 0.846677 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9890 t_0.4-0.6=0.8798 t_0.6-0.8=0.7089 t_0.8-1.0=0.6471] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 212/499 (42% done) --- + [Epoch 212][10/40] loss=0.887337 avg=0.869466 VRAM=39.2GiB | 42.4% done | ETA(epoch): 693s + [Epoch 212][20/40] loss=0.997133 avg=0.872960 VRAM=39.0GiB | 42.5% done | ETA(epoch): 462s + [Epoch 212][30/40] loss=0.884528 avg=0.861945 VRAM=39.2GiB | 42.5% done | ETA(epoch): 231s + [Epoch 212][40/40] loss=0.739158 avg=0.861645 VRAM=39.0GiB | 42.6% done | ETA(epoch): 0s + Train loss: 0.861645 (924.6s) ETA: 4816min + Val loss: 0.858972 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9882 t_0.4-0.6=0.8440 t_0.6-0.8=0.6709 t_0.8-1.0=0.6309] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 213/499 (43% done) --- + [Epoch 213][10/40] loss=0.853471 avg=0.849584 VRAM=39.2GiB | 42.6% done | ETA(epoch): 694s + [Epoch 213][20/40] loss=0.824144 avg=0.851256 VRAM=39.0GiB | 42.7% done | ETA(epoch): 462s + [Epoch 213][30/40] loss=0.872121 avg=0.853609 VRAM=39.2GiB | 42.8% done | ETA(epoch): 231s + [Epoch 213][40/40] loss=0.828777 avg=0.848658 VRAM=39.0GiB | 42.8% done | ETA(epoch): 0s + Train loss: 0.848658 (924.8s) ETA: 4799min + Val loss: 0.848380 [t_0.0-0.2=1.0918 t_0.2-0.4=1.0329 t_0.4-0.6=0.7750 t_0.6-0.8=0.7011 t_0.8-1.0=0.6429] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 214/499 (43% done) --- + [Epoch 214][10/40] loss=0.853402 avg=0.856429 VRAM=39.2GiB | 42.9% done | ETA(epoch): 693s + [Epoch 214][20/40] loss=0.926731 avg=0.849900 VRAM=39.0GiB | 42.9% done | ETA(epoch): 462s + [Epoch 214][30/40] loss=0.821618 avg=0.838860 VRAM=39.2GiB | 43.0% done | ETA(epoch): 231s + [Epoch 214][40/40] loss=0.894733 avg=0.839407 VRAM=39.0GiB | 43.0% done | ETA(epoch): 0s + Train loss: 0.839407 (924.7s) ETA: 4782min + Val loss: 0.854446 [t_0.0-0.2=1.0811 t_0.2-0.4=0.9843 t_0.4-0.6=0.8405 t_0.6-0.8=0.6886 t_0.8-1.0=0.6497] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 215/499 (43% done) --- + [Epoch 215][10/40] loss=0.868836 avg=0.835046 VRAM=39.2GiB | 43.0% done | ETA(epoch): 694s + [Epoch 215][20/40] loss=0.963597 avg=0.853403 VRAM=39.0GiB | 43.1% done | ETA(epoch): 462s + [Epoch 215][30/40] loss=0.804015 avg=0.849636 VRAM=39.2GiB | 43.1% done | ETA(epoch): 231s + [Epoch 215][40/40] loss=0.875048 avg=0.846646 VRAM=39.0GiB | 43.2% done | ETA(epoch): 0s + Train loss: 0.846646 (924.6s) ETA: 4765min + Val loss: 0.835130 [t_0.0-0.2=1.0652 t_0.2-0.4=0.9814 t_0.4-0.6=0.8377 t_0.6-0.8=0.7199 t_0.8-1.0=0.6248] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 216/499 (43% done) --- + [Epoch 216][10/40] loss=0.852936 avg=0.842540 VRAM=39.2GiB | 43.2% done | ETA(epoch): 693s + [Epoch 216][20/40] loss=0.866037 avg=0.845621 VRAM=39.0GiB | 43.3% done | ETA(epoch): 462s + [Epoch 216][30/40] loss=0.895376 avg=0.856588 VRAM=39.2GiB | 43.4% done | ETA(epoch): 231s + [Epoch 216][40/40] loss=0.785987 avg=0.855564 VRAM=39.0GiB | 43.4% done | ETA(epoch): 0s + Train loss: 0.855564 (924.2s) ETA: 4748min + Val loss: 0.844600 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9790 t_0.4-0.6=0.8441 t_0.6-0.8=0.7362 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 217/499 (43% done) --- + [Epoch 217][10/40] loss=0.775149 avg=0.892000 VRAM=39.2GiB | 43.5% done | ETA(epoch): 693s + [Epoch 217][20/40] loss=0.827530 avg=0.881292 VRAM=39.0GiB | 43.5% done | ETA(epoch): 462s + [Epoch 217][30/40] loss=0.922773 avg=0.882429 VRAM=39.2GiB | 43.5% done | ETA(epoch): 231s + [Epoch 217][40/40] loss=0.895517 avg=0.877262 VRAM=39.0GiB | 43.6% done | ETA(epoch): 0s + Train loss: 0.877262 (924.0s) ETA: 4731min + Val loss: 0.851170 [t_0.0-0.2=1.0623 t_0.2-0.4=0.9720 t_0.4-0.6=0.8600 t_0.6-0.8=0.7258 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 218/499 (44% done) --- + [Epoch 218][10/40] loss=0.898146 avg=0.879031 VRAM=39.2GiB | 43.6% done | ETA(epoch): 693s + [Epoch 218][20/40] loss=0.781500 avg=0.859036 VRAM=39.0GiB | 43.7% done | ETA(epoch): 462s + [Epoch 218][30/40] loss=0.812348 avg=0.863464 VRAM=39.2GiB | 43.8% done | ETA(epoch): 231s + [Epoch 218][40/40] loss=0.864048 avg=0.865514 VRAM=39.0GiB | 43.8% done | ETA(epoch): 0s + Train loss: 0.865514 (924.1s) ETA: 4713min + Val loss: 0.826487 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9977 t_0.4-0.6=0.8122 t_0.6-0.8=0.7275 t_0.8-1.0=0.6180] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 219/499 (44% done) --- + [Epoch 219][10/40] loss=0.878718 avg=0.836753 VRAM=39.2GiB | 43.9% done | ETA(epoch): 693s + [Epoch 219][20/40] loss=0.802485 avg=0.860193 VRAM=39.0GiB | 43.9% done | ETA(epoch): 462s + [Epoch 219][30/40] loss=0.782281 avg=0.854853 VRAM=39.2GiB | 44.0% done | ETA(epoch): 231s + [Epoch 219][40/40] loss=0.797592 avg=0.851952 VRAM=39.0GiB | 44.0% done | ETA(epoch): 0s + Train loss: 0.851952 (924.2s) ETA: 4696min + Val loss: 0.861681 [t_0.0-0.2=1.0674 t_0.2-0.4=0.9866 t_0.4-0.6=0.8921 t_0.6-0.8=0.7352 t_0.8-1.0=0.6341] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 220/499 (44% done) --- + [Epoch 220][10/40] loss=0.851338 avg=0.826837 VRAM=39.2GiB | 44.0% done | ETA(epoch): 693s + [Epoch 220][20/40] loss=0.925817 avg=0.851677 VRAM=39.0GiB | 44.1% done | ETA(epoch): 462s + [Epoch 220][30/40] loss=0.884926 avg=0.851705 VRAM=39.2GiB | 44.1% done | ETA(epoch): 231s + [Epoch 220][40/40] loss=0.750115 avg=0.844469 VRAM=39.0GiB | 44.2% done | ETA(epoch): 0s + Train loss: 0.844469 (924.1s) ETA: 4679min + Val loss: 0.847796 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9624 t_0.4-0.6=0.8678 t_0.6-0.8=0.7052 t_0.8-1.0=0.6390] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 221/499 (44% done) --- + [Epoch 221][10/40] loss=0.819361 avg=0.854603 VRAM=39.2GiB | 44.2% done | ETA(epoch): 693s + [Epoch 221][20/40] loss=0.823232 avg=0.844037 VRAM=39.0GiB | 44.3% done | ETA(epoch): 462s + [Epoch 221][30/40] loss=0.873460 avg=0.849500 VRAM=39.2GiB | 44.4% done | ETA(epoch): 231s + [Epoch 221][40/40] loss=0.878470 avg=0.848840 VRAM=39.0GiB | 44.4% done | ETA(epoch): 0s + Train loss: 0.848840 (923.9s) ETA: 4662min + Val loss: 0.856931 [t_0.0-0.2=1.0838 t_0.2-0.4=0.9729 t_0.4-0.6=0.8952 t_0.6-0.8=0.7157 t_0.8-1.0=0.6457] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 222/499 (44% done) --- + [Epoch 222][10/40] loss=0.877722 avg=0.864985 VRAM=39.2GiB | 44.5% done | ETA(epoch): 693s + [Epoch 222][20/40] loss=0.864611 avg=0.862536 VRAM=39.0GiB | 44.5% done | ETA(epoch): 462s + [Epoch 222][30/40] loss=0.776366 avg=0.864990 VRAM=39.2GiB | 44.5% done | ETA(epoch): 231s + [Epoch 222][40/40] loss=0.786495 avg=0.857417 VRAM=39.0GiB | 44.6% done | ETA(epoch): 0s + Train loss: 0.857417 (924.3s) ETA: 4645min + Val loss: 0.845345 [t_0.0-0.2=1.0552 t_0.2-0.4=0.9942 t_0.4-0.6=0.8411 t_0.6-0.8=0.7173 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 223/499 (45% done) --- + [Epoch 223][10/40] loss=0.828519 avg=0.826830 VRAM=39.2GiB | 44.6% done | ETA(epoch): 693s + [Epoch 223][20/40] loss=0.892403 avg=0.857102 VRAM=39.0GiB | 44.7% done | ETA(epoch): 462s + [Epoch 223][30/40] loss=0.869067 avg=0.854269 VRAM=39.2GiB | 44.8% done | ETA(epoch): 231s + [Epoch 223][40/40] loss=0.916090 avg=0.862836 VRAM=39.0GiB | 44.8% done | ETA(epoch): 0s + Train loss: 0.862836 (925.1s) ETA: 4628min + Val loss: 0.811804 [t_0.0-0.2=1.0475 t_0.2-0.4=0.9546 t_0.4-0.6=0.8313 t_0.6-0.8=0.7143 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 224/499 (45% done) --- + [Epoch 224][10/40] loss=0.811791 avg=0.831864 VRAM=39.2GiB | 44.9% done | ETA(epoch): 693s + [Epoch 224][20/40] loss=0.829106 avg=0.852093 VRAM=39.0GiB | 44.9% done | ETA(epoch): 462s + [Epoch 224][30/40] loss=0.801867 avg=0.858078 VRAM=39.2GiB | 45.0% done | ETA(epoch): 231s + [Epoch 224][40/40] loss=0.932386 avg=0.860043 VRAM=39.0GiB | 45.0% done | ETA(epoch): 0s + Train loss: 0.860043 (925.1s) ETA: 4611min + Val loss: 0.861263 [t_0.0-0.2=1.0824 t_0.2-0.4=1.0307 t_0.4-0.6=0.8685 t_0.6-0.8=0.7006 t_0.8-1.0=0.6183] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 225/499 (45% done) --- + [Epoch 225][10/40] loss=0.743190 avg=0.811464 VRAM=39.2GiB | 45.1% done | ETA(epoch): 694s + [Epoch 225][20/40] loss=0.855220 avg=0.830687 VRAM=39.0GiB | 45.1% done | ETA(epoch): 462s + [Epoch 225][30/40] loss=0.850606 avg=0.835077 VRAM=39.2GiB | 45.1% done | ETA(epoch): 231s + [Epoch 225][40/40] loss=0.929154 avg=0.843820 VRAM=39.0GiB | 45.2% done | ETA(epoch): 0s + Train loss: 0.843820 (924.4s) ETA: 4594min + Val loss: 0.872338 [t_0.0-0.2=1.0820 t_0.2-0.4=1.0146 t_0.4-0.6=0.8034 t_0.6-0.8=0.7147 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 226/499 (45% done) --- + [Epoch 226][10/40] loss=0.774009 avg=0.866530 VRAM=39.2GiB | 45.2% done | ETA(epoch): 693s + [Epoch 226][20/40] loss=0.800561 avg=0.854169 VRAM=39.0GiB | 45.3% done | ETA(epoch): 462s + [Epoch 226][30/40] loss=0.816020 avg=0.856864 VRAM=39.2GiB | 45.4% done | ETA(epoch): 231s + [Epoch 226][40/40] loss=0.745849 avg=0.855974 VRAM=39.0GiB | 45.4% done | ETA(epoch): 0s + Train loss: 0.855974 (924.6s) ETA: 4577min + Val loss: 0.879030 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9893 t_0.4-0.6=0.8465 t_0.6-0.8=0.7062 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 227/499 (45% done) --- + [Epoch 227][10/40] loss=0.949265 avg=0.862907 VRAM=39.2GiB | 45.5% done | ETA(epoch): 693s + [Epoch 227][20/40] loss=0.858898 avg=0.852964 VRAM=39.0GiB | 45.5% done | ETA(epoch): 462s + [Epoch 227][30/40] loss=0.844040 avg=0.859376 VRAM=39.2GiB | 45.6% done | ETA(epoch): 231s + [Epoch 227][40/40] loss=0.860302 avg=0.857435 VRAM=39.0GiB | 45.6% done | ETA(epoch): 0s + Train loss: 0.857435 (924.6s) ETA: 4560min + Val loss: 0.838970 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9788 t_0.4-0.6=0.8262 t_0.6-0.8=0.6966 t_0.8-1.0=0.6363] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 228/499 (46% done) --- + [Epoch 228][10/40] loss=0.761629 avg=0.843654 VRAM=39.2GiB | 45.6% done | ETA(epoch): 693s + [Epoch 228][20/40] loss=0.864152 avg=0.856280 VRAM=39.0GiB | 45.7% done | ETA(epoch): 462s + [Epoch 228][30/40] loss=0.883192 avg=0.859969 VRAM=39.2GiB | 45.8% done | ETA(epoch): 231s + [Epoch 228][40/40] loss=0.861483 avg=0.862632 VRAM=39.0GiB | 45.8% done | ETA(epoch): 0s + Train loss: 0.862632 (925.3s) ETA: 4543min + Val loss: 0.863722 [t_0.0-0.2=1.0771 t_0.2-0.4=0.9490 t_0.4-0.6=0.8535 t_0.6-0.8=0.7401 t_0.8-1.0=0.6450] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 20.0/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 229/499 (46% done) --- + [Epoch 229][10/40] loss=0.774477 avg=0.852639 VRAM=39.2GiB | 45.9% done | ETA(epoch): 694s + [Epoch 229][20/40] loss=0.877866 avg=0.849291 VRAM=39.0GiB | 45.9% done | ETA(epoch): 463s + [Epoch 229][30/40] loss=0.735637 avg=0.846809 VRAM=39.2GiB | 46.0% done | ETA(epoch): 231s + [Epoch 229][40/40] loss=0.804458 avg=0.840687 VRAM=39.0GiB | 46.0% done | ETA(epoch): 0s + Train loss: 0.840687 (925.2s) ETA: 4526min + Val loss: 0.825459 [t_0.0-0.2=1.0845 t_0.2-0.4=0.9486 t_0.4-0.6=0.7996 t_0.6-0.8=0.7074 t_0.8-1.0=0.6305] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 20.1/188.4 GiB (10.7%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 230/499 (46% done) --- + [Epoch 230][10/40] loss=0.809974 avg=0.854009 VRAM=39.2GiB | 46.1% done | ETA(epoch): 694s + [Epoch 230][20/40] loss=0.878691 avg=0.856289 VRAM=39.0GiB | 46.1% done | ETA(epoch): 463s + [Epoch 230][30/40] loss=0.738529 avg=0.853622 VRAM=39.2GiB | 46.2% done | ETA(epoch): 231s + [Epoch 230][40/40] loss=0.882421 avg=0.852632 VRAM=39.0GiB | 46.2% done | ETA(epoch): 0s + Train loss: 0.852632 (925.5s) ETA: 4508min + Val loss: 0.871450 [t_0.0-0.2=1.0622 t_0.2-0.4=1.0148 t_0.4-0.6=0.8112 t_0.6-0.8=0.7070 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 231/499 (46% done) --- + [Epoch 231][10/40] loss=0.789270 avg=0.862675 VRAM=39.2GiB | 46.2% done | ETA(epoch): 694s + [Epoch 231][20/40] loss=0.822337 avg=0.843894 VRAM=39.0GiB | 46.3% done | ETA(epoch): 463s + [Epoch 231][30/40] loss=0.897580 avg=0.850886 VRAM=39.2GiB | 46.4% done | ETA(epoch): 231s + [Epoch 231][40/40] loss=0.845039 avg=0.842824 VRAM=39.0GiB | 46.4% done | ETA(epoch): 0s + Train loss: 0.842824 (925.7s) ETA: 4491min + Val loss: 0.842361 [t_0.0-0.2=1.0755 t_0.2-0.4=0.9943 t_0.4-0.6=0.8675 t_0.6-0.8=0.7021 t_0.8-1.0=0.6388] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 232/499 (46% done) --- + [Epoch 232][10/40] loss=0.937815 avg=0.823316 VRAM=39.2GiB | 46.5% done | ETA(epoch): 694s + [Epoch 232][20/40] loss=0.733562 avg=0.823459 VRAM=39.0GiB | 46.5% done | ETA(epoch): 463s + [Epoch 232][30/40] loss=0.776897 avg=0.842867 VRAM=39.2GiB | 46.6% done | ETA(epoch): 231s + [Epoch 232][40/40] loss=0.814035 avg=0.846881 VRAM=39.0GiB | 46.6% done | ETA(epoch): 0s + Train loss: 0.846881 (925.6s) ETA: 4474min + Val loss: 0.845140 [t_0.0-0.2=1.0920 t_0.2-0.4=1.0321 t_0.4-0.6=0.8578 t_0.6-0.8=0.6964 t_0.8-1.0=0.6360] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 233/499 (47% done) --- + [Epoch 233][10/40] loss=0.825548 avg=0.830528 VRAM=39.2GiB | 46.7% done | ETA(epoch): 694s + [Epoch 233][20/40] loss=0.869651 avg=0.851926 VRAM=39.0GiB | 46.7% done | ETA(epoch): 462s + [Epoch 233][30/40] loss=0.827729 avg=0.853635 VRAM=39.2GiB | 46.8% done | ETA(epoch): 231s + [Epoch 233][40/40] loss=0.830377 avg=0.853384 VRAM=39.0GiB | 46.8% done | ETA(epoch): 0s + Train loss: 0.853384 (924.7s) ETA: 4457min + Val loss: 0.820927 [t_0.0-0.2=1.0717 t_0.2-0.4=1.0197 t_0.4-0.6=0.8163 t_0.6-0.8=0.7085 t_0.8-1.0=0.6238] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 234/499 (47% done) --- + [Epoch 234][10/40] loss=0.865781 avg=0.845672 VRAM=39.2GiB | 46.9% done | ETA(epoch): 693s + [Epoch 234][20/40] loss=0.841398 avg=0.851276 VRAM=39.0GiB | 46.9% done | ETA(epoch): 462s + [Epoch 234][30/40] loss=0.898137 avg=0.853542 VRAM=39.2GiB | 46.9% done | ETA(epoch): 231s + [Epoch 234][40/40] loss=0.935651 avg=0.848058 VRAM=39.0GiB | 47.0% done | ETA(epoch): 0s + Train loss: 0.848058 (923.6s) ETA: 4440min + Val loss: 0.820412 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9829 t_0.4-0.6=0.8345 t_0.6-0.8=0.7222 t_0.8-1.0=0.6347] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 235/499 (47% done) --- + [Epoch 235][10/40] loss=0.882296 avg=0.833227 VRAM=39.2GiB | 47.0% done | ETA(epoch): 693s + [Epoch 235][20/40] loss=0.951413 avg=0.842581 VRAM=39.0GiB | 47.1% done | ETA(epoch): 462s + [Epoch 235][30/40] loss=0.806376 avg=0.850433 VRAM=39.2GiB | 47.1% done | ETA(epoch): 231s + [Epoch 235][40/40] loss=0.713262 avg=0.847591 VRAM=39.0GiB | 47.2% done | ETA(epoch): 0s + Train loss: 0.847591 (923.9s) ETA: 4423min + Val loss: 0.875557 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9858 t_0.4-0.6=0.8655 t_0.6-0.8=0.7102 t_0.8-1.0=0.6367] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 236/499 (47% done) --- + [Epoch 236][10/40] loss=0.865717 avg=0.837189 VRAM=39.2GiB | 47.2% done | ETA(epoch): 693s + [Epoch 236][20/40] loss=0.829463 avg=0.841532 VRAM=39.0GiB | 47.3% done | ETA(epoch): 462s + [Epoch 236][30/40] loss=0.831971 avg=0.825577 VRAM=39.2GiB | 47.3% done | ETA(epoch): 231s + [Epoch 236][40/40] loss=0.860984 avg=0.833540 VRAM=39.0GiB | 47.4% done | ETA(epoch): 0s + Train loss: 0.833540 (924.0s) ETA: 4406min + Val loss: 0.827212 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9805 t_0.4-0.6=0.8641 t_0.6-0.8=0.7143 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 237/499 (47% done) --- + [Epoch 237][10/40] loss=0.775510 avg=0.840712 VRAM=39.2GiB | 47.4% done | ETA(epoch): 693s + [Epoch 237][20/40] loss=0.812101 avg=0.854561 VRAM=39.0GiB | 47.5% done | ETA(epoch): 462s + [Epoch 237][30/40] loss=0.834955 avg=0.851926 VRAM=39.2GiB | 47.5% done | ETA(epoch): 231s + [Epoch 237][40/40] loss=0.903152 avg=0.850073 VRAM=39.0GiB | 47.6% done | ETA(epoch): 0s + Train loss: 0.850073 (924.1s) ETA: 4389min + Val loss: 0.878169 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0043 t_0.4-0.6=0.8634 t_0.6-0.8=0.7065 t_0.8-1.0=0.6332] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 238/499 (48% done) --- + [Epoch 238][10/40] loss=0.910291 avg=0.852388 VRAM=39.2GiB | 47.6% done | ETA(epoch): 693s + [Epoch 238][20/40] loss=0.802685 avg=0.841491 VRAM=39.0GiB | 47.7% done | ETA(epoch): 462s + [Epoch 238][30/40] loss=0.946572 avg=0.845429 VRAM=39.2GiB | 47.8% done | ETA(epoch): 231s + [Epoch 238][40/40] loss=0.859711 avg=0.846530 VRAM=39.0GiB | 47.8% done | ETA(epoch): 0s + Train loss: 0.846530 (924.0s) ETA: 4372min + Val loss: 0.881523 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9634 t_0.4-0.6=0.8463 t_0.6-0.8=0.7323 t_0.8-1.0=0.6438] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 239/499 (48% done) --- + [Epoch 239][10/40] loss=0.908188 avg=0.841382 VRAM=39.2GiB | 47.9% done | ETA(epoch): 692s + [Epoch 239][20/40] loss=0.811418 avg=0.843741 VRAM=39.0GiB | 47.9% done | ETA(epoch): 462s + [Epoch 239][30/40] loss=0.801413 avg=0.840125 VRAM=39.2GiB | 47.9% done | ETA(epoch): 231s + [Epoch 239][40/40] loss=0.870170 avg=0.839959 VRAM=39.0GiB | 48.0% done | ETA(epoch): 0s + Train loss: 0.839959 (923.9s) ETA: 4355min + Val loss: 0.879646 [t_0.0-0.2=1.0860 t_0.2-0.4=0.9779 t_0.4-0.6=0.8753 t_0.6-0.8=0.7139 t_0.8-1.0=0.6458] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 240/499 (48% done) --- + [Epoch 240][10/40] loss=0.885435 avg=0.849003 VRAM=39.2GiB | 48.0% done | ETA(epoch): 693s + [Epoch 240][20/40] loss=0.887053 avg=0.858884 VRAM=39.0GiB | 48.1% done | ETA(epoch): 462s + [Epoch 240][30/40] loss=0.860669 avg=0.864227 VRAM=39.2GiB | 48.1% done | ETA(epoch): 231s + [Epoch 240][40/40] loss=0.789418 avg=0.859908 VRAM=39.0GiB | 48.2% done | ETA(epoch): 0s + Train loss: 0.859908 (923.7s) ETA: 4338min + Val loss: 0.842561 [t_0.0-0.2=1.0741 t_0.2-0.4=0.9637 t_0.4-0.6=0.8746 t_0.6-0.8=0.7158 t_0.8-1.0=0.6415] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 241/499 (48% done) --- + [Epoch 241][10/40] loss=0.856654 avg=0.902810 VRAM=39.2GiB | 48.2% done | ETA(epoch): 692s + [Epoch 241][20/40] loss=0.775597 avg=0.870597 VRAM=39.0GiB | 48.3% done | ETA(epoch): 461s + [Epoch 241][30/40] loss=0.805757 avg=0.868050 VRAM=39.2GiB | 48.4% done | ETA(epoch): 231s + [Epoch 241][40/40] loss=0.830607 avg=0.860212 VRAM=39.0GiB | 48.4% done | ETA(epoch): 0s + Train loss: 0.860212 (922.9s) ETA: 4321min + Val loss: 0.890412 [t_0.0-0.2=1.0668 t_0.2-0.4=0.9818 t_0.4-0.6=0.8607 t_0.6-0.8=0.7149 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 242/499 (48% done) --- + [Epoch 242][10/40] loss=0.980614 avg=0.840417 VRAM=39.2GiB | 48.4% done | ETA(epoch): 692s + [Epoch 242][20/40] loss=0.836815 avg=0.865860 VRAM=39.0GiB | 48.5% done | ETA(epoch): 461s + [Epoch 242][30/40] loss=0.812910 avg=0.859758 VRAM=39.2GiB | 48.5% done | ETA(epoch): 231s + [Epoch 242][40/40] loss=0.824374 avg=0.852448 VRAM=39.0GiB | 48.6% done | ETA(epoch): 0s + Train loss: 0.852448 (922.2s) ETA: 4304min + Val loss: 0.882320 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9760 t_0.4-0.6=0.8738 t_0.6-0.8=0.6888 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 243/499 (49% done) --- + [Epoch 243][10/40] loss=0.785417 avg=0.869409 VRAM=39.2GiB | 48.6% done | ETA(epoch): 692s + [Epoch 243][20/40] loss=0.822807 avg=0.866896 VRAM=39.0GiB | 48.7% done | ETA(epoch): 461s + [Epoch 243][30/40] loss=0.883839 avg=0.853612 VRAM=39.2GiB | 48.8% done | ETA(epoch): 231s + [Epoch 243][40/40] loss=0.771256 avg=0.848907 VRAM=39.0GiB | 48.8% done | ETA(epoch): 0s + Train loss: 0.848907 (922.1s) ETA: 4287min + Val loss: 0.851006 [t_0.0-0.2=1.0793 t_0.2-0.4=0.9786 t_0.4-0.6=0.8583 t_0.6-0.8=0.6986 t_0.8-1.0=0.6205] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 20.1/188.4 GiB (10.7%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 244/499 (49% done) --- + [Epoch 244][10/40] loss=0.817262 avg=0.841291 VRAM=39.2GiB | 48.9% done | ETA(epoch): 691s + [Epoch 244][20/40] loss=1.010666 avg=0.851522 VRAM=39.0GiB | 48.9% done | ETA(epoch): 461s + [Epoch 244][30/40] loss=0.977127 avg=0.856409 VRAM=39.2GiB | 48.9% done | ETA(epoch): 230s + [Epoch 244][40/40] loss=0.816253 avg=0.857537 VRAM=39.0GiB | 49.0% done | ETA(epoch): 0s + Train loss: 0.857537 (921.7s) ETA: 4270min + Val loss: 0.828345 [t_0.0-0.2=1.0729 t_0.2-0.4=0.9908 t_0.4-0.6=0.8414 t_0.6-0.8=0.7242 t_0.8-1.0=0.6304] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 245/499 (49% done) --- + [Epoch 245][10/40] loss=0.923168 avg=0.871280 VRAM=39.2GiB | 49.0% done | ETA(epoch): 691s + [Epoch 245][20/40] loss=0.873613 avg=0.862512 VRAM=39.0GiB | 49.1% done | ETA(epoch): 461s + [Epoch 245][30/40] loss=0.847291 avg=0.868473 VRAM=39.2GiB | 49.1% done | ETA(epoch): 230s + [Epoch 245][40/40] loss=0.896543 avg=0.859933 VRAM=39.0GiB | 49.2% done | ETA(epoch): 0s + Train loss: 0.859933 (921.4s) ETA: 4253min + Val loss: 0.844437 [t_0.0-0.2=1.0620 t_0.2-0.4=0.9973 t_0.4-0.6=0.8710 t_0.6-0.8=0.7032 t_0.8-1.0=0.6311] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0245 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 246/499 (49% done) --- + [Epoch 246][10/40] loss=0.849197 avg=0.856472 VRAM=39.2GiB | 49.2% done | ETA(epoch): 691s + [Epoch 246][20/40] loss=0.891697 avg=0.867309 VRAM=39.0GiB | 49.3% done | ETA(epoch): 461s + [Epoch 246][30/40] loss=0.792598 avg=0.842977 VRAM=39.2GiB | 49.4% done | ETA(epoch): 230s + [Epoch 246][40/40] loss=0.751568 avg=0.846830 VRAM=39.0GiB | 49.4% done | ETA(epoch): 0s + Train loss: 0.846830 (921.5s) ETA: 4236min + Val loss: 0.823175 [t_0.0-0.2=1.0727 t_0.2-0.4=0.9674 t_0.4-0.6=0.8346 t_0.6-0.8=0.7181 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 247/499 (49% done) --- + [Epoch 247][10/40] loss=0.922913 avg=0.832104 VRAM=39.2GiB | 49.5% done | ETA(epoch): 691s + [Epoch 247][20/40] loss=0.768216 avg=0.841707 VRAM=39.0GiB | 49.5% done | ETA(epoch): 461s + [Epoch 247][30/40] loss=0.743001 avg=0.839099 VRAM=39.2GiB | 49.5% done | ETA(epoch): 230s + [Epoch 247][40/40] loss=0.884142 avg=0.836552 VRAM=39.0GiB | 49.6% done | ETA(epoch): 0s + Train loss: 0.836552 (921.8s) ETA: 4219min + Val loss: 0.831057 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9879 t_0.4-0.6=0.8550 t_0.6-0.8=0.6781 t_0.8-1.0=0.6294] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 248/499 (50% done) --- + [Epoch 248][10/40] loss=0.851713 avg=0.860817 VRAM=39.2GiB | 49.6% done | ETA(epoch): 692s + [Epoch 248][20/40] loss=0.874296 avg=0.854828 VRAM=39.0GiB | 49.7% done | ETA(epoch): 461s + [Epoch 248][30/40] loss=0.876233 avg=0.850012 VRAM=39.2GiB | 49.8% done | ETA(epoch): 231s + [Epoch 248][40/40] loss=0.898515 avg=0.852758 VRAM=39.0GiB | 49.8% done | ETA(epoch): 0s + Train loss: 0.852758 (922.6s) ETA: 4202min + Val loss: 0.834065 [t_0.0-0.2=1.0838 t_0.2-0.4=1.0271 t_0.4-0.6=0.8203 t_0.6-0.8=0.6995 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0248 + Deleted old checkpoint: checkpoint_epoch_0245 +[MEM @ epoch 248 end] RAM: 18.5/188.4 GiB (9.8%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 249/499 (50% done) --- + [Epoch 249][10/40] loss=0.849714 avg=0.846651 VRAM=39.2GiB | 49.9% done | ETA(epoch): 692s + [Epoch 249][20/40] loss=0.846365 avg=0.830112 VRAM=39.0GiB | 49.9% done | ETA(epoch): 461s + [Epoch 249][30/40] loss=0.901042 avg=0.831644 VRAM=39.2GiB | 50.0% done | ETA(epoch): 231s + [Epoch 249][40/40] loss=0.881247 avg=0.839956 VRAM=39.0GiB | 50.0% done | ETA(epoch): 0s + Train loss: 0.839956 (922.9s) ETA: 4185min + Val loss: 0.854574 [t_0.0-0.2=1.0753 t_0.2-0.4=0.9723 t_0.4-0.6=0.8382 t_0.6-0.8=0.7149 t_0.8-1.0=0.6257] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 250/499 (50% done) --- + [MilestoneVis] train_0 step 10000 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 10000 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 10000 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 10000 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 10000 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 10000 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 10000 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 10000 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 10000 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 10000 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 250][10/40] loss=0.837118 avg=0.857378 VRAM=39.2GiB | 50.0% done | ETA(epoch): 2950s + [Epoch 250][20/40] loss=0.940466 avg=0.848406 VRAM=39.0GiB | 50.1% done | ETA(epoch): 1214s + [Epoch 250][30/40] loss=0.875434 avg=0.865439 VRAM=39.2GiB | 50.1% done | ETA(epoch): 482s + [Epoch 250][40/40] loss=0.887787 avg=0.866487 VRAM=39.0GiB | 50.2% done | ETA(epoch): 0s + Train loss: 0.866487 (1676.1s) ETA: 4180min + Val loss: 0.861227 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9756 t_0.4-0.6=0.8268 t_0.6-0.8=0.7013 t_0.8-1.0=0.6212] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 251/499 (50% done) --- + [Epoch 251][10/40] loss=0.893368 avg=0.824169 VRAM=39.2GiB | 50.2% done | ETA(epoch): 692s + [Epoch 251][20/40] loss=0.877024 avg=0.833718 VRAM=39.0GiB | 50.3% done | ETA(epoch): 462s + [Epoch 251][30/40] loss=0.931325 avg=0.843394 VRAM=39.2GiB | 50.3% done | ETA(epoch): 231s + [Epoch 251][40/40] loss=0.734167 avg=0.843746 VRAM=39.0GiB | 50.4% done | ETA(epoch): 0s + Train loss: 0.843746 (923.3s) ETA: 4163min + Val loss: 0.856995 [t_0.0-0.2=1.0762 t_0.2-0.4=1.0005 t_0.4-0.6=0.8178 t_0.6-0.8=0.6997 t_0.8-1.0=0.6292] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 252/499 (50% done) --- + [Epoch 252][10/40] loss=0.844173 avg=0.832579 VRAM=39.2GiB | 50.4% done | ETA(epoch): 692s + [Epoch 252][20/40] loss=0.807689 avg=0.844684 VRAM=39.0GiB | 50.5% done | ETA(epoch): 462s + [Epoch 252][30/40] loss=0.784360 avg=0.835919 VRAM=39.2GiB | 50.5% done | ETA(epoch): 231s + [Epoch 252][40/40] loss=0.829158 avg=0.841876 VRAM=39.0GiB | 50.6% done | ETA(epoch): 0s + Train loss: 0.841876 (923.5s) ETA: 4146min + Val loss: 0.829927 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9846 t_0.4-0.6=0.8302 t_0.6-0.8=0.7114 t_0.8-1.0=0.6179] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 19.1/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 253/499 (51% done) --- + [Epoch 253][10/40] loss=0.889590 avg=0.819238 VRAM=39.2GiB | 50.6% done | ETA(epoch): 692s + [Epoch 253][20/40] loss=0.898517 avg=0.833981 VRAM=39.0GiB | 50.7% done | ETA(epoch): 462s + [Epoch 253][30/40] loss=0.875370 avg=0.845607 VRAM=39.2GiB | 50.7% done | ETA(epoch): 231s + [Epoch 253][40/40] loss=0.805461 avg=0.850864 VRAM=39.0GiB | 50.8% done | ETA(epoch): 0s + Train loss: 0.850864 (923.7s) ETA: 4129min + Val loss: 0.844080 [t_0.0-0.2=1.0749 t_0.2-0.4=1.0099 t_0.4-0.6=0.8163 t_0.6-0.8=0.7212 t_0.8-1.0=0.6402] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 254/499 (51% done) --- + [Epoch 254][10/40] loss=0.940311 avg=0.854280 VRAM=39.2GiB | 50.8% done | ETA(epoch): 692s + [Epoch 254][20/40] loss=0.930257 avg=0.862432 VRAM=39.0GiB | 50.9% done | ETA(epoch): 462s + [Epoch 254][30/40] loss=0.757759 avg=0.856242 VRAM=39.2GiB | 50.9% done | ETA(epoch): 231s + [Epoch 254][40/40] loss=0.768702 avg=0.850934 VRAM=39.0GiB | 51.0% done | ETA(epoch): 0s + Train loss: 0.850934 (924.0s) ETA: 4112min + Val loss: 0.862289 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9902 t_0.4-0.6=0.8410 t_0.6-0.8=0.6962 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 255/499 (51% done) --- + [Epoch 255][10/40] loss=0.843607 avg=0.833730 VRAM=39.2GiB | 51.0% done | ETA(epoch): 693s + [Epoch 255][20/40] loss=0.949325 avg=0.850394 VRAM=39.0GiB | 51.1% done | ETA(epoch): 462s + [Epoch 255][30/40] loss=0.912812 avg=0.839840 VRAM=39.2GiB | 51.1% done | ETA(epoch): 231s + [Epoch 255][40/40] loss=0.954207 avg=0.841168 VRAM=39.0GiB | 51.2% done | ETA(epoch): 0s + Train loss: 0.841168 (923.7s) ETA: 4095min + Val loss: 0.854636 [t_0.0-0.2=1.0810 t_0.2-0.4=0.9752 t_0.4-0.6=0.8633 t_0.6-0.8=0.7013 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 19.1/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 256/499 (51% done) --- + [Epoch 256][10/40] loss=0.881277 avg=0.868362 VRAM=39.2GiB | 51.2% done | ETA(epoch): 692s + [Epoch 256][20/40] loss=0.735664 avg=0.869302 VRAM=39.0GiB | 51.3% done | ETA(epoch): 462s + [Epoch 256][30/40] loss=0.909356 avg=0.883276 VRAM=39.2GiB | 51.3% done | ETA(epoch): 231s + [Epoch 256][40/40] loss=0.880016 avg=0.882410 VRAM=39.0GiB | 51.4% done | ETA(epoch): 0s + Train loss: 0.882410 (923.6s) ETA: 4078min + Val loss: 0.828085 [t_0.0-0.2=1.0516 t_0.2-0.4=1.0386 t_0.4-0.6=0.8419 t_0.6-0.8=0.7128 t_0.8-1.0=0.6270] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 257/499 (51% done) --- + [Epoch 257][10/40] loss=0.953028 avg=0.860488 VRAM=39.2GiB | 51.4% done | ETA(epoch): 693s + [Epoch 257][20/40] loss=0.892600 avg=0.862689 VRAM=39.0GiB | 51.5% done | ETA(epoch): 462s + [Epoch 257][30/40] loss=0.832475 avg=0.856907 VRAM=39.2GiB | 51.5% done | ETA(epoch): 231s + [Epoch 257][40/40] loss=0.956016 avg=0.855397 VRAM=39.0GiB | 51.6% done | ETA(epoch): 0s + Train loss: 0.855397 (923.4s) ETA: 4061min + Val loss: 0.859847 [t_0.0-0.2=1.0574 t_0.2-0.4=0.9838 t_0.4-0.6=0.8666 t_0.6-0.8=0.7010 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 258/499 (52% done) --- + [Epoch 258][10/40] loss=0.892892 avg=0.872774 VRAM=39.2GiB | 51.6% done | ETA(epoch): 693s + [Epoch 258][20/40] loss=0.780124 avg=0.871327 VRAM=39.0GiB | 51.7% done | ETA(epoch): 462s + [Epoch 258][30/40] loss=0.808272 avg=0.849981 VRAM=39.2GiB | 51.7% done | ETA(epoch): 231s + [Epoch 258][40/40] loss=0.840008 avg=0.848310 VRAM=39.0GiB | 51.8% done | ETA(epoch): 0s + Train loss: 0.848310 (923.6s) ETA: 4044min + Val loss: 0.874750 [t_0.0-0.2=1.0689 t_0.2-0.4=0.9928 t_0.4-0.6=0.8522 t_0.6-0.8=0.7245 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 259/499 (52% done) --- + [Epoch 259][10/40] loss=0.894778 avg=0.852428 VRAM=39.2GiB | 51.8% done | ETA(epoch): 693s + [Epoch 259][20/40] loss=0.800852 avg=0.860455 VRAM=39.0GiB | 51.9% done | ETA(epoch): 462s + [Epoch 259][30/40] loss=0.864666 avg=0.859742 VRAM=39.2GiB | 51.9% done | ETA(epoch): 231s + [Epoch 259][40/40] loss=0.828037 avg=0.853518 VRAM=39.0GiB | 52.0% done | ETA(epoch): 0s + Train loss: 0.853518 (923.4s) ETA: 4027min + Val loss: 0.859141 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0068 t_0.4-0.6=0.8472 t_0.6-0.8=0.7238 t_0.8-1.0=0.6225] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 19.1/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 260/499 (52% done) --- + [Epoch 260][10/40] loss=0.749980 avg=0.819078 VRAM=39.2GiB | 52.0% done | ETA(epoch): 692s + [Epoch 260][20/40] loss=0.828950 avg=0.848885 VRAM=39.0GiB | 52.1% done | ETA(epoch): 461s + [Epoch 260][30/40] loss=0.826728 avg=0.840730 VRAM=39.2GiB | 52.1% done | ETA(epoch): 231s + [Epoch 260][40/40] loss=0.846000 avg=0.842660 VRAM=39.0GiB | 52.2% done | ETA(epoch): 0s + Train loss: 0.842660 (923.1s) ETA: 4010min + Val loss: 0.829894 [t_0.0-0.2=1.0727 t_0.2-0.4=1.0113 t_0.4-0.6=0.8534 t_0.6-0.8=0.6896 t_0.8-1.0=0.6218] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 261/499 (52% done) --- + [Epoch 261][10/40] loss=0.860957 avg=0.864694 VRAM=39.2GiB | 52.2% done | ETA(epoch): 692s + [Epoch 261][20/40] loss=0.914097 avg=0.864166 VRAM=39.0GiB | 52.3% done | ETA(epoch): 462s + [Epoch 261][30/40] loss=0.793699 avg=0.858955 VRAM=39.2GiB | 52.3% done | ETA(epoch): 231s + [Epoch 261][40/40] loss=0.926914 avg=0.860959 VRAM=39.0GiB | 52.4% done | ETA(epoch): 0s + Train loss: 0.860959 (923.5s) ETA: 3993min + Val loss: 0.828996 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9711 t_0.4-0.6=0.8140 t_0.6-0.8=0.7060 t_0.8-1.0=0.6269] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 262/499 (52% done) --- + [Epoch 262][10/40] loss=0.864492 avg=0.833411 VRAM=39.2GiB | 52.4% done | ETA(epoch): 692s + [Epoch 262][20/40] loss=0.940572 avg=0.843754 VRAM=39.0GiB | 52.5% done | ETA(epoch): 462s + [Epoch 262][30/40] loss=0.846596 avg=0.844638 VRAM=39.2GiB | 52.5% done | ETA(epoch): 231s + [Epoch 262][40/40] loss=0.863815 avg=0.840005 VRAM=39.0GiB | 52.6% done | ETA(epoch): 0s + Train loss: 0.840005 (923.3s) ETA: 3976min + Val loss: 0.856831 [t_0.0-0.2=1.0561 t_0.2-0.4=1.0223 t_0.4-0.6=0.8475 t_0.6-0.8=0.7029 t_0.8-1.0=0.6225] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 263/499 (53% done) --- + [Epoch 263][10/40] loss=0.849232 avg=0.855575 VRAM=39.2GiB | 52.6% done | ETA(epoch): 693s + [Epoch 263][20/40] loss=0.977981 avg=0.839824 VRAM=39.0GiB | 52.7% done | ETA(epoch): 462s + [Epoch 263][30/40] loss=0.838313 avg=0.840308 VRAM=39.2GiB | 52.8% done | ETA(epoch): 231s + [Epoch 263][40/40] loss=0.883659 avg=0.840494 VRAM=39.0GiB | 52.8% done | ETA(epoch): 0s + Train loss: 0.840494 (923.5s) ETA: 3959min + Val loss: 0.844626 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9851 t_0.4-0.6=0.8471 t_0.6-0.8=0.7061 t_0.8-1.0=0.6317] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 264/499 (53% done) --- + [Epoch 264][10/40] loss=0.934385 avg=0.887390 VRAM=39.2GiB | 52.8% done | ETA(epoch): 693s + [Epoch 264][20/40] loss=0.747436 avg=0.876948 VRAM=39.0GiB | 52.9% done | ETA(epoch): 462s + [Epoch 264][30/40] loss=0.831161 avg=0.876353 VRAM=39.2GiB | 52.9% done | ETA(epoch): 231s + [Epoch 264][40/40] loss=0.806094 avg=0.874516 VRAM=39.0GiB | 53.0% done | ETA(epoch): 0s + Train loss: 0.874516 (923.7s) ETA: 3942min + Val loss: 0.871561 [t_0.0-0.2=1.0522 t_0.2-0.4=0.9842 t_0.4-0.6=0.8572 t_0.6-0.8=0.7175 t_0.8-1.0=0.6217] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 265/499 (53% done) --- + [Epoch 265][10/40] loss=0.894777 avg=0.838876 VRAM=39.2GiB | 53.0% done | ETA(epoch): 693s + [Epoch 265][20/40] loss=0.893803 avg=0.849771 VRAM=39.0GiB | 53.1% done | ETA(epoch): 462s + [Epoch 265][30/40] loss=0.854961 avg=0.844769 VRAM=39.2GiB | 53.1% done | ETA(epoch): 231s + [Epoch 265][40/40] loss=0.822417 avg=0.844811 VRAM=39.0GiB | 53.2% done | ETA(epoch): 0s + Train loss: 0.844811 (923.9s) ETA: 3925min + Val loss: 0.833903 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9938 t_0.4-0.6=0.8261 t_0.6-0.8=0.7369 t_0.8-1.0=0.6188] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 266/499 (53% done) --- + [Epoch 266][10/40] loss=0.797325 avg=0.822575 VRAM=39.2GiB | 53.2% done | ETA(epoch): 693s + [Epoch 266][20/40] loss=0.925238 avg=0.847032 VRAM=39.0GiB | 53.3% done | ETA(epoch): 462s + [Epoch 266][30/40] loss=0.799115 avg=0.841489 VRAM=39.2GiB | 53.3% done | ETA(epoch): 231s + [Epoch 266][40/40] loss=0.821515 avg=0.840072 VRAM=39.0GiB | 53.4% done | ETA(epoch): 0s + Train loss: 0.840072 (923.8s) ETA: 3908min + Val loss: 0.867633 [t_0.0-0.2=1.0590 t_0.2-0.4=1.0003 t_0.4-0.6=0.8364 t_0.6-0.8=0.7356 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 267/499 (53% done) --- + [Epoch 267][10/40] loss=0.713855 avg=0.846406 VRAM=39.2GiB | 53.4% done | ETA(epoch): 693s + [Epoch 267][20/40] loss=0.851342 avg=0.839986 VRAM=39.0GiB | 53.5% done | ETA(epoch): 462s + [Epoch 267][30/40] loss=0.903302 avg=0.837888 VRAM=39.2GiB | 53.5% done | ETA(epoch): 231s + [Epoch 267][40/40] loss=0.869356 avg=0.842193 VRAM=39.0GiB | 53.6% done | ETA(epoch): 0s + Train loss: 0.842193 (924.3s) ETA: 3891min + Val loss: 0.833270 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9546 t_0.4-0.6=0.8620 t_0.6-0.8=0.7231 t_0.8-1.0=0.6409] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 19.1/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 268/499 (54% done) --- + [Epoch 268][10/40] loss=0.762239 avg=0.844785 VRAM=39.2GiB | 53.6% done | ETA(epoch): 693s + [Epoch 268][20/40] loss=0.912393 avg=0.848624 VRAM=39.0GiB | 53.7% done | ETA(epoch): 462s + [Epoch 268][30/40] loss=0.979573 avg=0.849104 VRAM=39.2GiB | 53.8% done | ETA(epoch): 231s + [Epoch 268][40/40] loss=0.874335 avg=0.842193 VRAM=39.0GiB | 53.8% done | ETA(epoch): 0s + Train loss: 0.842193 (924.2s) ETA: 3874min + Val loss: 0.855319 [t_0.0-0.2=1.0573 t_0.2-0.4=0.9896 t_0.4-0.6=0.8568 t_0.6-0.8=0.6945 t_0.8-1.0=0.6237] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 269/499 (54% done) --- + [Epoch 269][10/40] loss=0.917700 avg=0.828134 VRAM=39.2GiB | 53.8% done | ETA(epoch): 693s + [Epoch 269][20/40] loss=0.854966 avg=0.847891 VRAM=39.0GiB | 53.9% done | ETA(epoch): 462s + [Epoch 269][30/40] loss=0.892883 avg=0.845263 VRAM=39.2GiB | 53.9% done | ETA(epoch): 231s + [Epoch 269][40/40] loss=0.859168 avg=0.846489 VRAM=39.0GiB | 54.0% done | ETA(epoch): 0s + Train loss: 0.846489 (924.2s) ETA: 3857min + Val loss: 0.849315 [t_0.0-0.2=1.0703 t_0.2-0.4=0.9672 t_0.4-0.6=0.8573 t_0.6-0.8=0.7076 t_0.8-1.0=0.6398] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 270/499 (54% done) --- + [Epoch 270][10/40] loss=0.927954 avg=0.860716 VRAM=39.2GiB | 54.0% done | ETA(epoch): 693s + [Epoch 270][20/40] loss=0.941637 avg=0.859977 VRAM=39.0GiB | 54.1% done | ETA(epoch): 462s + [Epoch 270][30/40] loss=0.980850 avg=0.862057 VRAM=39.2GiB | 54.1% done | ETA(epoch): 231s + [Epoch 270][40/40] loss=0.933742 avg=0.859470 VRAM=39.0GiB | 54.2% done | ETA(epoch): 0s + Train loss: 0.859470 (924.0s) ETA: 3840min + Val loss: 0.859391 [t_0.0-0.2=1.0621 t_0.2-0.4=0.9688 t_0.4-0.6=0.8621 t_0.6-0.8=0.7066 t_0.8-1.0=0.6090] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 19.1/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 271/499 (54% done) --- + [Epoch 271][10/40] loss=0.826481 avg=0.854322 VRAM=39.2GiB | 54.2% done | ETA(epoch): 693s + [Epoch 271][20/40] loss=0.958453 avg=0.863486 VRAM=39.0GiB | 54.3% done | ETA(epoch): 462s + [Epoch 271][30/40] loss=0.893330 avg=0.863191 VRAM=39.2GiB | 54.4% done | ETA(epoch): 231s + [Epoch 271][40/40] loss=0.845720 avg=0.861739 VRAM=39.0GiB | 54.4% done | ETA(epoch): 0s + Train loss: 0.861739 (923.4s) ETA: 3823min + Val loss: 0.840564 [t_0.0-0.2=1.0787 t_0.2-0.4=1.0291 t_0.4-0.6=0.7832 t_0.6-0.8=0.6879 t_0.8-1.0=0.6347] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 272/499 (54% done) --- + [Epoch 272][10/40] loss=0.782840 avg=0.859552 VRAM=39.2GiB | 54.4% done | ETA(epoch): 693s + [Epoch 272][20/40] loss=0.833748 avg=0.850691 VRAM=39.0GiB | 54.5% done | ETA(epoch): 462s + [Epoch 272][30/40] loss=0.920807 avg=0.848477 VRAM=39.2GiB | 54.5% done | ETA(epoch): 231s + [Epoch 272][40/40] loss=0.788386 avg=0.846786 VRAM=39.0GiB | 54.6% done | ETA(epoch): 0s + Train loss: 0.846786 (924.0s) ETA: 3806min + Val loss: 0.862530 [t_0.0-0.2=1.0655 t_0.2-0.4=1.0161 t_0.4-0.6=0.8273 t_0.6-0.8=0.7030 t_0.8-1.0=0.6235] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 273/499 (55% done) --- + [Epoch 273][10/40] loss=0.863171 avg=0.852713 VRAM=39.2GiB | 54.6% done | ETA(epoch): 693s + [Epoch 273][20/40] loss=0.792822 avg=0.855442 VRAM=39.0GiB | 54.7% done | ETA(epoch): 462s + [Epoch 273][30/40] loss=0.923058 avg=0.870760 VRAM=39.2GiB | 54.8% done | ETA(epoch): 231s + [Epoch 273][40/40] loss=0.864694 avg=0.854294 VRAM=39.0GiB | 54.8% done | ETA(epoch): 0s + Train loss: 0.854294 (923.6s) ETA: 3789min + Val loss: 0.882035 [t_0.0-0.2=1.0671 t_0.2-0.4=0.9946 t_0.4-0.6=0.8405 t_0.6-0.8=0.7163 t_0.8-1.0=0.6234] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 274/499 (55% done) --- + [Epoch 274][10/40] loss=0.692194 avg=0.834437 VRAM=39.2GiB | 54.9% done | ETA(epoch): 692s + [Epoch 274][20/40] loss=0.900250 avg=0.822598 VRAM=39.0GiB | 54.9% done | ETA(epoch): 462s + [Epoch 274][30/40] loss=0.866491 avg=0.824660 VRAM=39.2GiB | 54.9% done | ETA(epoch): 231s + [Epoch 274][40/40] loss=0.837107 avg=0.835534 VRAM=39.0GiB | 55.0% done | ETA(epoch): 0s + Train loss: 0.835534 (923.5s) ETA: 3772min + Val loss: 0.869534 [t_0.0-0.2=1.0725 t_0.2-0.4=0.9561 t_0.4-0.6=0.8756 t_0.6-0.8=0.7129 t_0.8-1.0=0.6420] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 275/499 (55% done) --- + [Epoch 275][10/40] loss=0.871270 avg=0.844928 VRAM=39.2GiB | 55.0% done | ETA(epoch): 692s + [Epoch 275][20/40] loss=0.816353 avg=0.855780 VRAM=39.0GiB | 55.1% done | ETA(epoch): 462s + [Epoch 275][30/40] loss=0.837199 avg=0.850674 VRAM=39.2GiB | 55.1% done | ETA(epoch): 231s + [Epoch 275][40/40] loss=0.850636 avg=0.840803 VRAM=39.0GiB | 55.2% done | ETA(epoch): 0s + Train loss: 0.840803 (923.6s) ETA: 3755min + Val loss: 0.824907 [t_0.0-0.2=1.0693 t_0.2-0.4=0.9709 t_0.4-0.6=0.8632 t_0.6-0.8=0.6888 t_0.8-1.0=0.6367] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 276/499 (55% done) --- + [Epoch 276][10/40] loss=0.820584 avg=0.864597 VRAM=39.2GiB | 55.2% done | ETA(epoch): 693s + [Epoch 276][20/40] loss=0.812332 avg=0.861893 VRAM=39.0GiB | 55.3% done | ETA(epoch): 462s + [Epoch 276][30/40] loss=0.808554 avg=0.865542 VRAM=39.2GiB | 55.4% done | ETA(epoch): 231s + [Epoch 276][40/40] loss=0.873613 avg=0.868075 VRAM=39.0GiB | 55.4% done | ETA(epoch): 0s + Train loss: 0.868075 (924.0s) ETA: 3738min + Val loss: 0.827544 [t_0.0-0.2=1.0583 t_0.2-0.4=0.9961 t_0.4-0.6=0.8384 t_0.6-0.8=0.7148 t_0.8-1.0=0.6243] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 277/499 (55% done) --- + [Epoch 277][10/40] loss=0.812835 avg=0.863612 VRAM=39.2GiB | 55.5% done | ETA(epoch): 693s + [Epoch 277][20/40] loss=0.876561 avg=0.858667 VRAM=39.0GiB | 55.5% done | ETA(epoch): 462s + [Epoch 277][30/40] loss=0.838664 avg=0.857646 VRAM=39.2GiB | 55.5% done | ETA(epoch): 231s + [Epoch 277][40/40] loss=0.818586 avg=0.851881 VRAM=39.0GiB | 55.6% done | ETA(epoch): 0s + Train loss: 0.851881 (923.7s) ETA: 3721min + Val loss: 0.852557 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9886 t_0.4-0.6=0.8364 t_0.6-0.8=0.7136 t_0.8-1.0=0.6166] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 278/499 (56% done) --- + [Epoch 278][10/40] loss=0.905822 avg=0.850013 VRAM=39.2GiB | 55.6% done | ETA(epoch): 693s + [Epoch 278][20/40] loss=0.837308 avg=0.841515 VRAM=39.0GiB | 55.7% done | ETA(epoch): 462s + [Epoch 278][30/40] loss=0.912555 avg=0.855060 VRAM=39.2GiB | 55.8% done | ETA(epoch): 231s + [Epoch 278][40/40] loss=0.865250 avg=0.853867 VRAM=39.0GiB | 55.8% done | ETA(epoch): 0s + Train loss: 0.853867 (924.0s) ETA: 3704min + Val loss: 0.832931 [t_0.0-0.2=1.0757 t_0.2-0.4=1.0040 t_0.4-0.6=0.8456 t_0.6-0.8=0.6896 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 279/499 (56% done) --- + [Epoch 279][10/40] loss=0.854121 avg=0.834270 VRAM=39.2GiB | 55.9% done | ETA(epoch): 693s + [Epoch 279][20/40] loss=0.872853 avg=0.846370 VRAM=39.0GiB | 55.9% done | ETA(epoch): 462s + [Epoch 279][30/40] loss=0.889114 avg=0.849946 VRAM=39.2GiB | 56.0% done | ETA(epoch): 231s + [Epoch 279][40/40] loss=0.751513 avg=0.854944 VRAM=39.0GiB | 56.0% done | ETA(epoch): 0s + Train loss: 0.854944 (923.8s) ETA: 3687min + Val loss: 0.823375 [t_0.0-0.2=1.0708 t_0.2-0.4=0.9542 t_0.4-0.6=0.8591 t_0.6-0.8=0.7099 t_0.8-1.0=0.6318] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 280/499 (56% done) --- + [Epoch 280][10/40] loss=0.824418 avg=0.833092 VRAM=39.2GiB | 56.0% done | ETA(epoch): 693s + [Epoch 280][20/40] loss=0.795858 avg=0.826085 VRAM=39.0GiB | 56.1% done | ETA(epoch): 462s + [Epoch 280][30/40] loss=0.950367 avg=0.846850 VRAM=39.2GiB | 56.1% done | ETA(epoch): 231s + [Epoch 280][40/40] loss=0.918658 avg=0.856375 VRAM=39.0GiB | 56.2% done | ETA(epoch): 0s + Train loss: 0.856375 (923.7s) ETA: 3670min + Val loss: 0.851191 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9730 t_0.4-0.6=0.8515 t_0.6-0.8=0.6861 t_0.8-1.0=0.6261] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 281/499 (56% done) --- + [Epoch 281][10/40] loss=0.843323 avg=0.872601 VRAM=39.2GiB | 56.2% done | ETA(epoch): 693s + [Epoch 281][20/40] loss=0.834754 avg=0.872419 VRAM=39.0GiB | 56.3% done | ETA(epoch): 462s + [Epoch 281][30/40] loss=0.735329 avg=0.867119 VRAM=39.2GiB | 56.4% done | ETA(epoch): 231s + [Epoch 281][40/40] loss=0.835909 avg=0.864915 VRAM=39.0GiB | 56.4% done | ETA(epoch): 0s + Train loss: 0.864915 (924.1s) ETA: 3653min + Val loss: 0.855724 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0006 t_0.4-0.6=0.8056 t_0.6-0.8=0.7069 t_0.8-1.0=0.6389] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0281 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 282/499 (56% done) --- + [Epoch 282][10/40] loss=0.927208 avg=0.875315 VRAM=39.2GiB | 56.5% done | ETA(epoch): 693s + [Epoch 282][20/40] loss=0.904749 avg=0.863852 VRAM=39.0GiB | 56.5% done | ETA(epoch): 462s + [Epoch 282][30/40] loss=0.856355 avg=0.863506 VRAM=39.2GiB | 56.5% done | ETA(epoch): 231s + [Epoch 282][40/40] loss=0.922676 avg=0.862346 VRAM=39.0GiB | 56.6% done | ETA(epoch): 0s + Train loss: 0.862346 (923.5s) ETA: 3636min + Val loss: 0.845040 [t_0.0-0.2=1.0584 t_0.2-0.4=1.0235 t_0.4-0.6=0.8408 t_0.6-0.8=0.7044 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 283/499 (57% done) --- + [Epoch 283][10/40] loss=0.857249 avg=0.862099 VRAM=39.2GiB | 56.6% done | ETA(epoch): 692s + [Epoch 283][20/40] loss=0.765029 avg=0.856927 VRAM=39.0GiB | 56.7% done | ETA(epoch): 462s + [Epoch 283][30/40] loss=0.797445 avg=0.865192 VRAM=39.2GiB | 56.8% done | ETA(epoch): 231s + [Epoch 283][40/40] loss=0.814359 avg=0.860233 VRAM=39.0GiB | 56.8% done | ETA(epoch): 0s + Train loss: 0.860233 (923.6s) ETA: 3619min + Val loss: 0.833359 [t_0.0-0.2=1.0559 t_0.2-0.4=1.0224 t_0.4-0.6=0.8504 t_0.6-0.8=0.7048 t_0.8-1.0=0.6273] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 284/499 (57% done) --- + [Epoch 284][10/40] loss=0.916299 avg=0.848226 VRAM=39.2GiB | 56.9% done | ETA(epoch): 693s + [Epoch 284][20/40] loss=0.836350 avg=0.834683 VRAM=39.0GiB | 56.9% done | ETA(epoch): 462s + [Epoch 284][30/40] loss=0.722364 avg=0.838933 VRAM=39.2GiB | 57.0% done | ETA(epoch): 231s + [Epoch 284][40/40] loss=0.732173 avg=0.847589 VRAM=39.0GiB | 57.0% done | ETA(epoch): 0s + Train loss: 0.847589 (924.1s) ETA: 3602min + Val loss: 0.909350 [t_0.0-0.2=1.0611 t_0.2-0.4=0.9949 t_0.4-0.6=0.8515 t_0.6-0.8=0.7184 t_0.8-1.0=0.6115] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0284 + Deleted old checkpoint: checkpoint_epoch_0281 +[MEM @ epoch 284 end] RAM: 19.1/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 285/499 (57% done) --- + [Epoch 285][10/40] loss=0.859828 avg=0.859479 VRAM=39.2GiB | 57.0% done | ETA(epoch): 693s + [Epoch 285][20/40] loss=0.894539 avg=0.854389 VRAM=39.0GiB | 57.1% done | ETA(epoch): 462s + [Epoch 285][30/40] loss=0.914149 avg=0.861252 VRAM=39.2GiB | 57.1% done | ETA(epoch): 231s + [Epoch 285][40/40] loss=0.911086 avg=0.865756 VRAM=39.0GiB | 57.2% done | ETA(epoch): 0s + Train loss: 0.865756 (924.0s) ETA: 3585min + Val loss: 0.845909 [t_0.0-0.2=1.0905 t_0.2-0.4=0.9612 t_0.4-0.6=0.8149 t_0.6-0.8=0.7267 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 286/499 (57% done) --- + [Epoch 286][10/40] loss=0.922846 avg=0.835044 VRAM=39.2GiB | 57.2% done | ETA(epoch): 693s + [Epoch 286][20/40] loss=0.815224 avg=0.824389 VRAM=39.0GiB | 57.3% done | ETA(epoch): 462s + [Epoch 286][30/40] loss=0.851866 avg=0.834252 VRAM=39.2GiB | 57.4% done | ETA(epoch): 231s + [Epoch 286][40/40] loss=0.933657 avg=0.846846 VRAM=39.0GiB | 57.4% done | ETA(epoch): 0s + Train loss: 0.846846 (924.1s) ETA: 3568min + Val loss: 0.849402 [t_0.0-0.2=1.0662 t_0.2-0.4=0.9964 t_0.4-0.6=0.8365 t_0.6-0.8=0.7131 t_0.8-1.0=0.6276] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 19.1/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 287/499 (57% done) --- + [Epoch 287][10/40] loss=0.919838 avg=0.852993 VRAM=39.2GiB | 57.5% done | ETA(epoch): 693s + [Epoch 287][20/40] loss=0.839944 avg=0.839927 VRAM=39.0GiB | 57.5% done | ETA(epoch): 462s + [Epoch 287][30/40] loss=0.836022 avg=0.838831 VRAM=39.2GiB | 57.6% done | ETA(epoch): 231s + [Epoch 287][40/40] loss=0.860598 avg=0.841977 VRAM=39.0GiB | 57.6% done | ETA(epoch): 0s + Train loss: 0.841977 (924.2s) ETA: 3551min + Val loss: 0.801869 [t_0.0-0.2=1.0372 t_0.2-0.4=0.9784 t_0.4-0.6=0.8450 t_0.6-0.8=0.7289 t_0.8-1.0=0.6431] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0287 (BEST) + Deleted old checkpoint: checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 288/499 (58% done) --- + [Epoch 288][10/40] loss=0.893930 avg=0.828537 VRAM=39.2GiB | 57.6% done | ETA(epoch): 693s + [Epoch 288][20/40] loss=0.887893 avg=0.829658 VRAM=39.0GiB | 57.7% done | ETA(epoch): 462s + [Epoch 288][30/40] loss=0.803816 avg=0.841781 VRAM=39.2GiB | 57.8% done | ETA(epoch): 231s + [Epoch 288][40/40] loss=0.888773 avg=0.845156 VRAM=39.0GiB | 57.8% done | ETA(epoch): 0s + Train loss: 0.845156 (924.6s) ETA: 3534min + Val loss: 0.865580 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9709 t_0.4-0.6=0.8380 t_0.6-0.8=0.7058 t_0.8-1.0=0.6308] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 289/499 (58% done) --- + [Epoch 289][10/40] loss=0.807506 avg=0.847841 VRAM=39.2GiB | 57.9% done | ETA(epoch): 693s + [Epoch 289][20/40] loss=0.726184 avg=0.840379 VRAM=39.0GiB | 57.9% done | ETA(epoch): 462s + [Epoch 289][30/40] loss=0.732782 avg=0.854315 VRAM=39.2GiB | 58.0% done | ETA(epoch): 231s + [Epoch 289][40/40] loss=0.802540 avg=0.849362 VRAM=39.0GiB | 58.0% done | ETA(epoch): 0s + Train loss: 0.849362 (924.8s) ETA: 3517min + Val loss: 0.828752 [t_0.0-0.2=1.0651 t_0.2-0.4=0.9752 t_0.4-0.6=0.8632 t_0.6-0.8=0.7176 t_0.8-1.0=0.6333] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 290/499 (58% done) --- + [Epoch 290][10/40] loss=0.826051 avg=0.826559 VRAM=39.2GiB | 58.1% done | ETA(epoch): 694s + [Epoch 290][20/40] loss=0.913990 avg=0.849863 VRAM=39.0GiB | 58.1% done | ETA(epoch): 463s + [Epoch 290][30/40] loss=0.917364 avg=0.866456 VRAM=39.2GiB | 58.1% done | ETA(epoch): 231s + [Epoch 290][40/40] loss=1.002974 avg=0.862614 VRAM=39.0GiB | 58.2% done | ETA(epoch): 0s + Train loss: 0.862614 (924.7s) ETA: 3500min + Val loss: 0.849960 [t_0.0-0.2=1.0839 t_0.2-0.4=0.9929 t_0.4-0.6=0.8309 t_0.6-0.8=0.7126 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0290 +[MEM @ epoch 290 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 291/499 (58% done) --- + [Epoch 291][10/40] loss=0.775236 avg=0.860963 VRAM=39.2GiB | 58.2% done | ETA(epoch): 693s + [Epoch 291][20/40] loss=0.838780 avg=0.856647 VRAM=39.0GiB | 58.3% done | ETA(epoch): 462s + [Epoch 291][30/40] loss=0.820728 avg=0.855540 VRAM=39.2GiB | 58.4% done | ETA(epoch): 231s + [Epoch 291][40/40] loss=0.929762 avg=0.861888 VRAM=39.0GiB | 58.4% done | ETA(epoch): 0s + Train loss: 0.861888 (924.6s) ETA: 3483min + Val loss: 0.865839 [t_0.0-0.2=1.0717 t_0.2-0.4=1.0147 t_0.4-0.6=0.8470 t_0.6-0.8=0.6993 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 292/499 (58% done) --- + [Epoch 292][10/40] loss=0.926773 avg=0.834953 VRAM=39.2GiB | 58.5% done | ETA(epoch): 693s + [Epoch 292][20/40] loss=0.891169 avg=0.849363 VRAM=39.0GiB | 58.5% done | ETA(epoch): 462s + [Epoch 292][30/40] loss=0.904012 avg=0.857324 VRAM=39.2GiB | 58.6% done | ETA(epoch): 231s + [Epoch 292][40/40] loss=0.690819 avg=0.848332 VRAM=39.0GiB | 58.6% done | ETA(epoch): 0s + Train loss: 0.848332 (924.6s) ETA: 3466min + Val loss: 0.850837 [t_0.0-0.2=1.0547 t_0.2-0.4=1.0326 t_0.4-0.6=0.8261 t_0.6-0.8=0.6861 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 293/499 (59% done) --- + [Epoch 293][10/40] loss=0.928975 avg=0.877324 VRAM=39.2GiB | 58.7% done | ETA(epoch): 693s + [Epoch 293][20/40] loss=0.824179 avg=0.844630 VRAM=39.0GiB | 58.7% done | ETA(epoch): 462s + [Epoch 293][30/40] loss=0.892776 avg=0.849834 VRAM=39.2GiB | 58.8% done | ETA(epoch): 231s + [Epoch 293][40/40] loss=0.947478 avg=0.846959 VRAM=39.0GiB | 58.8% done | ETA(epoch): 0s + Train loss: 0.846959 (924.6s) ETA: 3450min + Val loss: 0.806321 [t_0.0-0.2=1.0821 t_0.2-0.4=0.9663 t_0.4-0.6=0.8631 t_0.6-0.8=0.6955 t_0.8-1.0=0.6199] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 294/499 (59% done) --- + [Epoch 294][10/40] loss=0.854080 avg=0.878609 VRAM=39.2GiB | 58.9% done | ETA(epoch): 693s + [Epoch 294][20/40] loss=0.776794 avg=0.862989 VRAM=39.0GiB | 58.9% done | ETA(epoch): 462s + [Epoch 294][30/40] loss=0.863177 avg=0.858833 VRAM=39.2GiB | 59.0% done | ETA(epoch): 231s + [Epoch 294][40/40] loss=0.929884 avg=0.860477 VRAM=39.0GiB | 59.0% done | ETA(epoch): 0s + Train loss: 0.860477 (924.6s) ETA: 3433min + Val loss: 0.876402 [t_0.0-0.2=1.0695 t_0.2-0.4=1.0202 t_0.4-0.6=0.8391 t_0.6-0.8=0.7164 t_0.8-1.0=0.6379] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 295/499 (59% done) --- + [Epoch 295][10/40] loss=0.912174 avg=0.842845 VRAM=39.2GiB | 59.1% done | ETA(epoch): 693s + [Epoch 295][20/40] loss=0.806181 avg=0.847153 VRAM=39.0GiB | 59.1% done | ETA(epoch): 462s + [Epoch 295][30/40] loss=0.827548 avg=0.845824 VRAM=39.2GiB | 59.2% done | ETA(epoch): 231s + [Epoch 295][40/40] loss=0.907308 avg=0.843491 VRAM=39.0GiB | 59.2% done | ETA(epoch): 0s + Train loss: 0.843491 (924.4s) ETA: 3416min + Val loss: 0.839664 [t_0.0-0.2=1.0737 t_0.2-0.4=1.0127 t_0.4-0.6=0.8330 t_0.6-0.8=0.7179 t_0.8-1.0=0.6267] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 296/499 (59% done) --- + [Epoch 296][10/40] loss=0.899234 avg=0.864074 VRAM=39.2GiB | 59.2% done | ETA(epoch): 693s + [Epoch 296][20/40] loss=0.782529 avg=0.859691 VRAM=39.0GiB | 59.3% done | ETA(epoch): 462s + [Epoch 296][30/40] loss=0.932615 avg=0.862285 VRAM=39.2GiB | 59.4% done | ETA(epoch): 231s + [Epoch 296][40/40] loss=0.826067 avg=0.853219 VRAM=39.0GiB | 59.4% done | ETA(epoch): 0s + Train loss: 0.853219 (924.4s) ETA: 3399min + Val loss: 0.855520 [t_0.0-0.2=1.0745 t_0.2-0.4=1.0167 t_0.4-0.6=0.8288 t_0.6-0.8=0.7060 t_0.8-1.0=0.6335] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 297/499 (59% done) --- + [Epoch 297][10/40] loss=0.865098 avg=0.837015 VRAM=39.2GiB | 59.5% done | ETA(epoch): 693s + [Epoch 297][20/40] loss=0.815732 avg=0.828977 VRAM=39.0GiB | 59.5% done | ETA(epoch): 462s + [Epoch 297][30/40] loss=0.878985 avg=0.837169 VRAM=39.2GiB | 59.6% done | ETA(epoch): 231s + [Epoch 297][40/40] loss=0.826324 avg=0.843056 VRAM=39.0GiB | 59.6% done | ETA(epoch): 0s + Train loss: 0.843056 (924.2s) ETA: 3382min + Val loss: 0.872335 [t_0.0-0.2=1.0651 t_0.2-0.4=1.0029 t_0.4-0.6=0.8760 t_0.6-0.8=0.7169 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 298/499 (60% done) --- + [Epoch 298][10/40] loss=0.908597 avg=0.866243 VRAM=39.2GiB | 59.7% done | ETA(epoch): 692s + [Epoch 298][20/40] loss=0.814837 avg=0.861003 VRAM=39.0GiB | 59.7% done | ETA(epoch): 461s + [Epoch 298][30/40] loss=0.985237 avg=0.856433 VRAM=39.2GiB | 59.8% done | ETA(epoch): 231s + [Epoch 298][40/40] loss=0.850102 avg=0.860172 VRAM=39.0GiB | 59.8% done | ETA(epoch): 0s + Train loss: 0.860172 (923.4s) ETA: 3365min + Val loss: 0.854887 [t_0.0-0.2=1.0721 t_0.2-0.4=0.9939 t_0.4-0.6=0.8244 t_0.6-0.8=0.7283 t_0.8-1.0=0.6355] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 299/499 (60% done) --- + [Epoch 299][10/40] loss=0.791214 avg=0.870945 VRAM=39.2GiB | 59.9% done | ETA(epoch): 693s + [Epoch 299][20/40] loss=0.817643 avg=0.869130 VRAM=39.0GiB | 59.9% done | ETA(epoch): 462s + [Epoch 299][30/40] loss=0.878201 avg=0.858923 VRAM=39.2GiB | 60.0% done | ETA(epoch): 231s + [Epoch 299][40/40] loss=0.844276 avg=0.856648 VRAM=39.0GiB | 60.0% done | ETA(epoch): 0s + [MilestoneVis] train_0 step 11999 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 11999 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 11999 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 11999 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 11999 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 11999 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 11999 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 11999 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 11999 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 11999 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_384_H200/milestone_vis/milestone_grid_test_5.png + Train loss: 0.856648 (1679.2s) ETA: 3357min + Val loss: 0.819186 [t_0.0-0.2=1.0726 t_0.2-0.4=1.0044 t_0.4-0.6=0.8260 t_0.6-0.8=0.7061 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 300/499 (60% done) --- + [Epoch 300][10/40] loss=0.896676 avg=0.851369 VRAM=39.2GiB | 60.1% done | ETA(epoch): 693s + [Epoch 300][20/40] loss=0.802761 avg=0.851783 VRAM=39.0GiB | 60.1% done | ETA(epoch): 462s + [Epoch 300][30/40] loss=0.919747 avg=0.852093 VRAM=39.2GiB | 60.2% done | ETA(epoch): 231s + [Epoch 300][40/40] loss=0.913792 avg=0.851666 VRAM=39.0GiB | 60.2% done | ETA(epoch): 0s + Train loss: 0.851666 (924.5s) ETA: 3340min + Val loss: 0.865224 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9665 t_0.4-0.6=0.8472 t_0.6-0.8=0.7019 t_0.8-1.0=0.6431] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0300 + Deleted old checkpoint: checkpoint_epoch_0297 +[MEM @ epoch 300 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 301/499 (60% done) --- + [Epoch 301][10/40] loss=0.952295 avg=0.846742 VRAM=39.2GiB | 60.2% done | ETA(epoch): 693s + [Epoch 301][20/40] loss=0.834164 avg=0.853996 VRAM=39.0GiB | 60.3% done | ETA(epoch): 462s + [Epoch 301][30/40] loss=0.907791 avg=0.849223 VRAM=39.2GiB | 60.4% done | ETA(epoch): 231s + [Epoch 301][40/40] loss=0.684310 avg=0.845554 VRAM=39.0GiB | 60.4% done | ETA(epoch): 0s + Train loss: 0.845554 (924.5s) ETA: 3323min + Val loss: 0.861252 [t_0.0-0.2=1.0591 t_0.2-0.4=1.0153 t_0.4-0.6=0.8568 t_0.6-0.8=0.7141 t_0.8-1.0=0.6438] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0301 + Deleted old checkpoint: checkpoint_epoch_0298 +[MEM @ epoch 301 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 302/499 (60% done) --- + [Epoch 302][10/40] loss=0.865171 avg=0.844497 VRAM=39.2GiB | 60.5% done | ETA(epoch): 693s + [Epoch 302][20/40] loss=0.856370 avg=0.845242 VRAM=39.0GiB | 60.5% done | ETA(epoch): 462s + [Epoch 302][30/40] loss=0.798026 avg=0.846714 VRAM=39.2GiB | 60.6% done | ETA(epoch): 231s + [Epoch 302][40/40] loss=0.769848 avg=0.847465 VRAM=39.0GiB | 60.6% done | ETA(epoch): 0s + Train loss: 0.847465 (924.8s) ETA: 3306min + Val loss: 0.868377 [t_0.0-0.2=1.0890 t_0.2-0.4=0.9870 t_0.4-0.6=0.8808 t_0.6-0.8=0.6830 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0302 + Deleted old checkpoint: checkpoint_epoch_0299 +[MEM @ epoch 302 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 303/499 (61% done) --- + [Epoch 303][10/40] loss=0.793993 avg=0.838841 VRAM=39.2GiB | 60.7% done | ETA(epoch): 693s + [Epoch 303][20/40] loss=0.824003 avg=0.864718 VRAM=39.0GiB | 60.7% done | ETA(epoch): 462s + [Epoch 303][30/40] loss=0.790955 avg=0.857730 VRAM=39.2GiB | 60.8% done | ETA(epoch): 231s + [Epoch 303][40/40] loss=0.824675 avg=0.856265 VRAM=39.0GiB | 60.8% done | ETA(epoch): 0s + Train loss: 0.856265 (924.3s) ETA: 3289min + Val loss: 0.844249 [t_0.0-0.2=1.0518 t_0.2-0.4=0.9919 t_0.4-0.6=0.8590 t_0.6-0.8=0.7080 t_0.8-1.0=0.6299] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0303 + Deleted old checkpoint: checkpoint_epoch_0300 +[MEM @ epoch 303 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 304/499 (61% done) --- + [Epoch 304][10/40] loss=0.897788 avg=0.846805 VRAM=39.2GiB | 60.9% done | ETA(epoch): 693s + [Epoch 304][20/40] loss=0.873480 avg=0.836071 VRAM=39.0GiB | 60.9% done | ETA(epoch): 462s + [Epoch 304][30/40] loss=0.966958 avg=0.852702 VRAM=39.2GiB | 61.0% done | ETA(epoch): 231s + [Epoch 304][40/40] loss=0.875123 avg=0.858737 VRAM=39.0GiB | 61.0% done | ETA(epoch): 0s + Train loss: 0.858737 (924.5s) ETA: 3272min + Val loss: 0.856214 [t_0.0-0.2=1.0834 t_0.2-0.4=0.9749 t_0.4-0.6=0.8477 t_0.6-0.8=0.6934 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0304 + Deleted old checkpoint: checkpoint_epoch_0301 +[MEM @ epoch 304 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 305/499 (61% done) --- + [Epoch 305][10/40] loss=0.692420 avg=0.845405 VRAM=39.2GiB | 61.1% done | ETA(epoch): 693s + [Epoch 305][20/40] loss=0.836292 avg=0.849996 VRAM=39.0GiB | 61.1% done | ETA(epoch): 462s + [Epoch 305][30/40] loss=0.811809 avg=0.857049 VRAM=39.2GiB | 61.2% done | ETA(epoch): 231s + [Epoch 305][40/40] loss=0.915079 avg=0.852711 VRAM=39.0GiB | 61.2% done | ETA(epoch): 0s + Train loss: 0.852711 (923.9s) ETA: 3255min + Val loss: 0.846410 [t_0.0-0.2=1.0778 t_0.2-0.4=0.9891 t_0.4-0.6=0.8355 t_0.6-0.8=0.7100 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0305 + Deleted old checkpoint: checkpoint_epoch_0302 +[MEM @ epoch 305 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 306/499 (61% done) --- + [Epoch 306][10/40] loss=0.856783 avg=0.863925 VRAM=39.2GiB | 61.3% done | ETA(epoch): 693s + [Epoch 306][20/40] loss=0.783212 avg=0.853582 VRAM=39.0GiB | 61.3% done | ETA(epoch): 462s + [Epoch 306][30/40] loss=0.918347 avg=0.855387 VRAM=39.2GiB | 61.4% done | ETA(epoch): 231s + [Epoch 306][40/40] loss=0.915482 avg=0.851240 VRAM=39.0GiB | 61.4% done | ETA(epoch): 0s + Train loss: 0.851240 (924.0s) ETA: 3238min + Val loss: 0.834797 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9983 t_0.4-0.6=0.8785 t_0.6-0.8=0.6926 t_0.8-1.0=0.6296] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0306 + Deleted old checkpoint: checkpoint_epoch_0303 +[MEM @ epoch 306 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 307/499 (61% done) --- + [Epoch 307][10/40] loss=0.817989 avg=0.840566 VRAM=39.2GiB | 61.5% done | ETA(epoch): 693s + [Epoch 307][20/40] loss=0.788078 avg=0.838850 VRAM=39.0GiB | 61.5% done | ETA(epoch): 462s + [Epoch 307][30/40] loss=0.804483 avg=0.836312 VRAM=39.2GiB | 61.6% done | ETA(epoch): 231s + [Epoch 307][40/40] loss=0.795373 avg=0.841337 VRAM=39.0GiB | 61.6% done | ETA(epoch): 0s + Train loss: 0.841337 (924.0s) ETA: 3221min + Val loss: 0.834734 [t_0.0-0.2=1.0729 t_0.2-0.4=1.0310 t_0.4-0.6=0.8212 t_0.6-0.8=0.7039 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0307 + Deleted old checkpoint: checkpoint_epoch_0304 +[MEM @ epoch 307 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 308/499 (62% done) --- + [Epoch 308][10/40] loss=0.903853 avg=0.843643 VRAM=39.2GiB | 61.7% done | ETA(epoch): 693s + [Epoch 308][20/40] loss=0.842743 avg=0.847571 VRAM=39.0GiB | 61.7% done | ETA(epoch): 462s + [Epoch 308][30/40] loss=0.839633 avg=0.856111 VRAM=39.2GiB | 61.8% done | ETA(epoch): 231s + [Epoch 308][40/40] loss=0.821534 avg=0.858838 VRAM=39.0GiB | 61.8% done | ETA(epoch): 0s + Train loss: 0.858838 (924.1s) ETA: 3204min + Val loss: 0.859801 [t_0.0-0.2=1.0729 t_0.2-0.4=0.9796 t_0.4-0.6=0.8547 t_0.6-0.8=0.7019 t_0.8-1.0=0.6372] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0308 + Deleted old checkpoint: checkpoint_epoch_0305 +[MEM @ epoch 308 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 309/499 (62% done) --- + [Epoch 309][10/40] loss=0.859497 avg=0.855182 VRAM=39.2GiB | 61.9% done | ETA(epoch): 693s + [Epoch 309][20/40] loss=0.856812 avg=0.859476 VRAM=39.0GiB | 61.9% done | ETA(epoch): 462s + [Epoch 309][30/40] loss=0.955118 avg=0.860657 VRAM=39.2GiB | 62.0% done | ETA(epoch): 231s + [Epoch 309][40/40] loss=0.914909 avg=0.851784 VRAM=39.0GiB | 62.0% done | ETA(epoch): 0s + Train loss: 0.851784 (923.9s) ETA: 3187min + Val loss: 0.876085 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9980 t_0.4-0.6=0.8288 t_0.6-0.8=0.7158 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0309 + Deleted old checkpoint: checkpoint_epoch_0306 +[MEM @ epoch 309 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 310/499 (62% done) --- + [Epoch 310][10/40] loss=0.900670 avg=0.849585 VRAM=39.2GiB | 62.1% done | ETA(epoch): 693s + [Epoch 310][20/40] loss=0.791668 avg=0.840173 VRAM=39.0GiB | 62.1% done | ETA(epoch): 462s + [Epoch 310][30/40] loss=0.889557 avg=0.855257 VRAM=39.2GiB | 62.2% done | ETA(epoch): 231s + [Epoch 310][40/40] loss=0.857232 avg=0.854876 VRAM=39.0GiB | 62.2% done | ETA(epoch): 0s + Train loss: 0.854876 (923.7s) ETA: 3170min + Val loss: 0.856162 [t_0.0-0.2=1.0763 t_0.2-0.4=0.9920 t_0.4-0.6=0.8348 t_0.6-0.8=0.7035 t_0.8-1.0=0.6387] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0310 + Deleted old checkpoint: checkpoint_epoch_0307 +[MEM @ epoch 310 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 311/499 (62% done) --- + [Epoch 311][10/40] loss=0.855982 avg=0.860293 VRAM=39.2GiB | 62.3% done | ETA(epoch): 692s + [Epoch 311][20/40] loss=0.922911 avg=0.868548 VRAM=39.0GiB | 62.3% done | ETA(epoch): 462s + [Epoch 311][30/40] loss=0.907615 avg=0.855605 VRAM=39.2GiB | 62.4% done | ETA(epoch): 231s + [Epoch 311][40/40] loss=0.919585 avg=0.860013 VRAM=39.0GiB | 62.4% done | ETA(epoch): 0s + Train loss: 0.860013 (923.9s) ETA: 3153min + Val loss: 0.875363 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0107 t_0.4-0.6=0.8695 t_0.6-0.8=0.6998 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0311 + Deleted old checkpoint: checkpoint_epoch_0308 +[MEM @ epoch 311 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 312/499 (62% done) --- + [Epoch 312][10/40] loss=0.888837 avg=0.852168 VRAM=39.2GiB | 62.5% done | ETA(epoch): 693s + [Epoch 312][20/40] loss=0.834634 avg=0.848325 VRAM=39.0GiB | 62.5% done | ETA(epoch): 462s + [Epoch 312][30/40] loss=0.776201 avg=0.853721 VRAM=39.2GiB | 62.5% done | ETA(epoch): 231s + [Epoch 312][40/40] loss=0.810349 avg=0.853786 VRAM=39.0GiB | 62.6% done | ETA(epoch): 0s + Train loss: 0.853786 (923.7s) ETA: 3136min + Val loss: 0.848976 [t_0.0-0.2=1.0733 t_0.2-0.4=0.9970 t_0.4-0.6=0.8273 t_0.6-0.8=0.6933 t_0.8-1.0=0.6135] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0312 + Deleted old checkpoint: checkpoint_epoch_0309 +[MEM @ epoch 312 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 313/499 (63% done) --- + [Epoch 313][10/40] loss=0.888605 avg=0.878318 VRAM=39.2GiB | 62.6% done | ETA(epoch): 692s + [Epoch 313][20/40] loss=0.840170 avg=0.871933 VRAM=39.0GiB | 62.7% done | ETA(epoch): 462s + [Epoch 313][30/40] loss=0.844765 avg=0.860467 VRAM=39.2GiB | 62.7% done | ETA(epoch): 231s + [Epoch 313][40/40] loss=0.848558 avg=0.850381 VRAM=39.0GiB | 62.8% done | ETA(epoch): 0s + Train loss: 0.850381 (923.0s) ETA: 3119min + Val loss: 0.861944 [t_0.0-0.2=1.0670 t_0.2-0.4=1.0040 t_0.4-0.6=0.8101 t_0.6-0.8=0.7356 t_0.8-1.0=0.6290] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0313 + Deleted old checkpoint: checkpoint_epoch_0310 +[MEM @ epoch 313 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 314/499 (63% done) --- + [Epoch 314][10/40] loss=0.898159 avg=0.831285 VRAM=39.2GiB | 62.8% done | ETA(epoch): 692s + [Epoch 314][20/40] loss=0.812436 avg=0.832904 VRAM=39.0GiB | 62.9% done | ETA(epoch): 462s + [Epoch 314][30/40] loss=0.776780 avg=0.833827 VRAM=39.2GiB | 62.9% done | ETA(epoch): 231s + [Epoch 314][40/40] loss=0.907444 avg=0.837411 VRAM=39.0GiB | 63.0% done | ETA(epoch): 0s + Train loss: 0.837411 (924.0s) ETA: 3102min + Val loss: 0.813823 [t_0.0-0.2=1.0622 t_0.2-0.4=0.9774 t_0.4-0.6=0.8266 t_0.6-0.8=0.7346 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0314 + Deleted old checkpoint: checkpoint_epoch_0311 +[MEM @ epoch 314 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 315/499 (63% done) --- + [Epoch 315][10/40] loss=0.956081 avg=0.837901 VRAM=39.2GiB | 63.0% done | ETA(epoch): 693s + [Epoch 315][20/40] loss=0.793885 avg=0.836653 VRAM=39.0GiB | 63.1% done | ETA(epoch): 462s + [Epoch 315][30/40] loss=0.843598 avg=0.844594 VRAM=39.2GiB | 63.1% done | ETA(epoch): 231s + [Epoch 315][40/40] loss=0.861109 avg=0.842892 VRAM=39.0GiB | 63.2% done | ETA(epoch): 0s + Train loss: 0.842892 (924.0s) ETA: 3085min + Val loss: 0.838884 [t_0.0-0.2=1.0502 t_0.2-0.4=0.9890 t_0.4-0.6=0.8428 t_0.6-0.8=0.7223 t_0.8-1.0=0.6222] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0315 + Deleted old checkpoint: checkpoint_epoch_0312 +[MEM @ epoch 315 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 316/499 (63% done) --- + [Epoch 316][10/40] loss=0.775152 avg=0.841509 VRAM=39.2GiB | 63.2% done | ETA(epoch): 693s + [Epoch 316][20/40] loss=0.776248 avg=0.831210 VRAM=39.0GiB | 63.3% done | ETA(epoch): 462s + [Epoch 316][30/40] loss=0.865084 avg=0.832990 VRAM=39.2GiB | 63.3% done | ETA(epoch): 231s + [Epoch 316][40/40] loss=0.881979 avg=0.838782 VRAM=39.0GiB | 63.4% done | ETA(epoch): 0s + Train loss: 0.838782 (924.1s) ETA: 3068min + Val loss: 0.848688 [t_0.0-0.2=1.0738 t_0.2-0.4=1.0185 t_0.4-0.6=0.8264 t_0.6-0.8=0.6902 t_0.8-1.0=0.6141] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0316 + Deleted old checkpoint: checkpoint_epoch_0313 +[MEM @ epoch 316 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 317/499 (63% done) --- + [Epoch 317][10/40] loss=0.810747 avg=0.842654 VRAM=39.2GiB | 63.4% done | ETA(epoch): 693s + [Epoch 317][20/40] loss=0.717405 avg=0.837956 VRAM=39.0GiB | 63.5% done | ETA(epoch): 462s + [Epoch 317][30/40] loss=0.727336 avg=0.837888 VRAM=39.2GiB | 63.5% done | ETA(epoch): 231s + [Epoch 317][40/40] loss=0.799849 avg=0.835752 VRAM=39.0GiB | 63.6% done | ETA(epoch): 0s + Train loss: 0.835752 (924.2s) ETA: 3052min + Val loss: 0.857647 [t_0.0-0.2=1.0797 t_0.2-0.4=0.9861 t_0.4-0.6=0.8082 t_0.6-0.8=0.7303 t_0.8-1.0=0.6199] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0317 + Deleted old checkpoint: checkpoint_epoch_0314 +[MEM @ epoch 317 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 318/499 (64% done) --- + [Epoch 318][10/40] loss=0.861912 avg=0.878219 VRAM=39.2GiB | 63.6% done | ETA(epoch): 693s + [Epoch 318][20/40] loss=0.798773 avg=0.863841 VRAM=39.0GiB | 63.7% done | ETA(epoch): 462s + [Epoch 318][30/40] loss=0.954517 avg=0.858730 VRAM=39.2GiB | 63.7% done | ETA(epoch): 231s + [Epoch 318][40/40] loss=0.831503 avg=0.853781 VRAM=39.0GiB | 63.8% done | ETA(epoch): 0s + Train loss: 0.853781 (924.3s) ETA: 3035min + Val loss: 0.859520 [t_0.0-0.2=1.0919 t_0.2-0.4=0.9808 t_0.4-0.6=0.8425 t_0.6-0.8=0.7246 t_0.8-1.0=0.6235] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0318 + Deleted old checkpoint: checkpoint_epoch_0315 +[MEM @ epoch 318 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 319/499 (64% done) --- + [Epoch 319][10/40] loss=0.788907 avg=0.828782 VRAM=39.2GiB | 63.8% done | ETA(epoch): 693s + [Epoch 319][20/40] loss=0.794482 avg=0.843898 VRAM=39.0GiB | 63.9% done | ETA(epoch): 462s + [Epoch 319][30/40] loss=0.809516 avg=0.852606 VRAM=39.2GiB | 63.9% done | ETA(epoch): 231s + [Epoch 319][40/40] loss=0.812185 avg=0.850841 VRAM=39.0GiB | 64.0% done | ETA(epoch): 0s + Train loss: 0.850841 (924.2s) ETA: 3018min + Val loss: 0.858154 [t_0.0-0.2=1.0713 t_0.2-0.4=1.0007 t_0.4-0.6=0.8480 t_0.6-0.8=0.7131 t_0.8-1.0=0.6325] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0319 + Deleted old checkpoint: checkpoint_epoch_0316 +[MEM @ epoch 319 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 320/499 (64% done) --- + [Epoch 320][10/40] loss=0.871283 avg=0.882286 VRAM=39.2GiB | 64.0% done | ETA(epoch): 693s + [Epoch 320][20/40] loss=0.797515 avg=0.860521 VRAM=39.0GiB | 64.1% done | ETA(epoch): 462s + [Epoch 320][30/40] loss=0.772822 avg=0.851003 VRAM=39.2GiB | 64.1% done | ETA(epoch): 231s + [Epoch 320][40/40] loss=0.760781 avg=0.847518 VRAM=39.0GiB | 64.2% done | ETA(epoch): 0s + Train loss: 0.847518 (923.8s) ETA: 3001min + Val loss: 0.865815 [t_0.0-0.2=1.0599 t_0.2-0.4=0.9991 t_0.4-0.6=0.8600 t_0.6-0.8=0.6922 t_0.8-1.0=0.6201] + Checkpoint saved: output/lora_256_dim_384_H200/checkpoint_epoch_0320 + Deleted old checkpoint: checkpoint_epoch_0317 +[MEM @ epoch 320 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.4/139.8 GiB (27.5%) + +--- Epoch 321/499 (64% done) --- diff --git a/training log/trainlog(lora256dim512) b/training log/trainlog(lora256dim512) new file mode 100644 index 0000000000000000000000000000000000000000..087b2c0f77a2d0187fd3e9487818f09a37e68720 --- /dev/null +++ b/training log/trainlog(lora256dim512) @@ -0,0 +1,4281 @@ + +====================================================================== +Command: train_script.py --name lora_256_dim_512_H100 --batch-size 6 --adapter-lr 1e-3 --lora-lr 1e-4 --adapter-mlp --lora-rank 256 --fusion-dim 512 --seed 42 +====================================================================== +Run dir : output/lora_256_dim_512_H100 +Log file: output/lora_256_dim_512_H100/train.log +GPU: NVIDIA H100 NVL | VRAM: 93.1 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: setting up run v3sc1sdx (0.2s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260423_231404-v3sc1sdx +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_256_dim_512_H100-train-20260423-231404 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_256_dim_512_H100 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_256_dim_512_H100/runs/v3sc1sdx + +Run Summary: + name lora_256_dim_512_H100 + fusion MLP (lightweight) + LoRA ON (rank=256) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_256_dim_512_H100 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 512 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 6 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_256_dim_512_H100 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.7/188.5 GiB (6.2%) | VRAM: 0.0/93.1 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H100 NVL (93.1 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.6s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 12.0/188.5 GiB (6.4%) | VRAM: 0.2/93.1 GiB (0.2%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (58.3s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 13.5/188.5 GiB (7.2%) | VRAM: 37.9/93.1 GiB (40.7%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 9.4M params +Control: 0.0M params +Total trainable: 9.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=256, alpha=256.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 78.64M + +Parameter Statistics: + HDC²A Adapter: total=9.4M trainable=9.4M + ControlNet (frozen): total=4212.2M LoRA trainable=78.64M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 9.4M + LoRA 78.64M = 88.09M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~8000 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0442, std=0.5195 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.430984 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.78 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.005310 + depth_encoder.conv_stem.6.weight: 0.004944 + W_s.weight: 0.003036 + semantic_encoder.conv_stem.3.weight: 0.002991 + semantic_encoder.proj.weight: 0.002960 + Test result: PASSED +[MEM @ after test] RAM: 14.0/188.5 GiB (7.4%) | VRAM: 38.1/93.1 GiB (40.9%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 672.097 GFLOPs + transformer: 273662.180 GFLOPs + total: 274334.277 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=6 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=6 +[Data] Test: 30 samples, batch_size=6 + +====================================================================== +Starting training: 500 epochs × 66 steps = 33000 total steps + batch_size=6, grad_accum=4, world_size=1, effective_bs=24 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 3300, 6600, 9900, 13200, 16500, 19799, 23099, 26399, 29699, 32999] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/499 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/66] loss=0.996021 avg=0.923983 VRAM=39.0GiB | 0.0% done | ETA(epoch): 6725s + [Epoch 0][20/66] loss=0.950910 avg=0.917009 VRAM=38.8GiB | 0.1% done | ETA(epoch): 3206s + [Epoch 0][30/66] loss=1.057543 avg=0.919208 VRAM=39.0GiB | 0.1% done | ETA(epoch): 1902s + [Epoch 0][40/66] loss=0.860612 avg=0.917848 VRAM=38.8GiB | 0.1% done | ETA(epoch): 1155s + [Epoch 0][50/66] loss=0.902975 avg=0.920852 VRAM=39.0GiB | 0.2% done | ETA(epoch): 630s + [Epoch 0][60/66] loss=0.951722 avg=0.918388 VRAM=38.8GiB | 0.2% done | ETA(epoch): 216s + Train loss: 0.919216 (2274.4s) ETA: 18953min + Val loss: 1.020531 [t_0.0-0.2=1.0888 t_0.2-0.4=1.0667 t_0.4-0.6=0.8857 t_0.6-0.8=0.7746 t_0.8-1.0=1.1116] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 1/499 (0% done) --- + [Epoch 1][10/66] loss=1.020646 avg=0.905561 VRAM=39.0GiB | 0.2% done | ETA(epoch): 1170s + [Epoch 1][20/66] loss=0.917925 avg=0.907168 VRAM=38.8GiB | 0.3% done | ETA(epoch): 920s + [Epoch 1][30/66] loss=0.968527 avg=0.910933 VRAM=39.0GiB | 0.3% done | ETA(epoch): 710s + [Epoch 1][40/66] loss=0.993664 avg=0.913719 VRAM=38.8GiB | 0.3% done | ETA(epoch): 509s + [Epoch 1][50/66] loss=0.870458 avg=0.918239 VRAM=39.0GiB | 0.4% done | ETA(epoch): 312s + [Epoch 1][60/66] loss=0.903457 avg=0.918072 VRAM=38.8GiB | 0.4% done | ETA(epoch): 117s + Train loss: 0.917160 (1280.6s) ETA: 15163min + Val loss: 0.908054 [t_0.0-0.2=1.0841 t_0.2-0.4=1.0273 t_0.4-0.6=0.8776 t_0.6-0.8=0.7673 t_0.8-1.0=0.9046] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 2/499 (0% done) --- + [Epoch 2][10/66] loss=0.931147 avg=0.911352 VRAM=39.0GiB | 0.4% done | ETA(epoch): 1070s + [Epoch 2][20/66] loss=0.938794 avg=0.915478 VRAM=38.8GiB | 0.5% done | ETA(epoch): 880s + [Epoch 2][30/66] loss=0.856924 avg=0.913370 VRAM=39.0GiB | 0.5% done | ETA(epoch): 689s + [Epoch 2][40/66] loss=0.883569 avg=0.917996 VRAM=38.8GiB | 0.5% done | ETA(epoch): 498s + [Epoch 2][50/66] loss=0.848230 avg=0.911157 VRAM=39.0GiB | 0.6% done | ETA(epoch): 306s + [Epoch 2][60/66] loss=0.855815 avg=0.905246 VRAM=38.8GiB | 0.6% done | ETA(epoch): 115s + Train loss: 0.905236 (1264.2s) ETA: 13859min + Val loss: 0.950950 [t_0.0-0.2=1.0971 t_0.2-0.4=1.0214 t_0.4-0.6=0.8328 t_0.6-0.8=0.7581 t_0.8-1.0=0.9908] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 3/499 (1% done) --- + [Epoch 3][10/66] loss=0.829429 avg=0.924085 VRAM=39.0GiB | 0.6% done | ETA(epoch): 1073s + [Epoch 3][20/66] loss=0.881167 avg=0.914141 VRAM=38.8GiB | 0.7% done | ETA(epoch): 883s + [Epoch 3][30/66] loss=0.905954 avg=0.907854 VRAM=39.0GiB | 0.7% done | ETA(epoch): 691s + [Epoch 3][40/66] loss=0.973570 avg=0.907615 VRAM=38.8GiB | 0.7% done | ETA(epoch): 499s + [Epoch 3][50/66] loss=0.870821 avg=0.903702 VRAM=39.0GiB | 0.8% done | ETA(epoch): 307s + [Epoch 3][60/66] loss=0.857071 avg=0.900882 VRAM=38.8GiB | 0.8% done | ETA(epoch): 115s + Train loss: 0.898488 (1265.1s) ETA: 13182min + Val loss: 0.888543 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9775 t_0.4-0.6=0.8575 t_0.6-0.8=0.7735 t_0.8-1.0=0.9326] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 4/499 (1% done) --- + [Epoch 4][10/66] loss=0.887449 avg=0.867496 VRAM=39.0GiB | 0.8% done | ETA(epoch): 1075s + [Epoch 4][20/66] loss=0.876555 avg=0.882984 VRAM=38.8GiB | 0.9% done | ETA(epoch): 882s + [Epoch 4][30/66] loss=0.890715 avg=0.881822 VRAM=39.0GiB | 0.9% done | ETA(epoch): 690s + [Epoch 4][40/66] loss=0.865174 avg=0.875283 VRAM=38.8GiB | 0.9% done | ETA(epoch): 498s + [Epoch 4][50/66] loss=0.894648 avg=0.884430 VRAM=39.0GiB | 1.0% done | ETA(epoch): 308s + [Epoch 4][60/66] loss=0.896396 avg=0.875710 VRAM=38.8GiB | 1.0% done | ETA(epoch): 115s + Train loss: 0.875862 (1267.7s) ETA: 12772min + Val loss: 0.936644 [t_0.0-0.2=1.0821 t_0.2-0.4=1.0147 t_0.4-0.6=0.8809 t_0.6-0.8=0.7369 t_0.8-1.0=0.9287] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 5/499 (1% done) --- + [Epoch 5][10/66] loss=0.822586 avg=0.879532 VRAM=39.0GiB | 1.0% done | ETA(epoch): 1071s + [Epoch 5][20/66] loss=0.898609 avg=0.880502 VRAM=38.8GiB | 1.1% done | ETA(epoch): 880s + [Epoch 5][30/66] loss=0.860028 avg=0.880420 VRAM=39.0GiB | 1.1% done | ETA(epoch): 689s + [Epoch 5][40/66] loss=0.986131 avg=0.891687 VRAM=38.8GiB | 1.1% done | ETA(epoch): 510s + [Epoch 5][50/66] loss=0.844808 avg=0.886074 VRAM=39.0GiB | 1.2% done | ETA(epoch): 312s + [Epoch 5][60/66] loss=0.826761 avg=0.883151 VRAM=38.8GiB | 1.2% done | ETA(epoch): 117s + Train loss: 0.884270 (1282.5s) ETA: 12511min + Val loss: 0.892923 [t_0.0-0.2=1.0857 t_0.2-0.4=0.9819 t_0.4-0.6=0.8356 t_0.6-0.8=0.7661 t_0.8-1.0=0.8510] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 6/499 (1% done) --- + [Epoch 6][10/66] loss=0.831961 avg=0.894636 VRAM=39.0GiB | 1.2% done | ETA(epoch): 1072s + [Epoch 6][20/66] loss=0.921910 avg=0.894761 VRAM=38.8GiB | 1.3% done | ETA(epoch): 880s + [Epoch 6][30/66] loss=0.883422 avg=0.893150 VRAM=39.0GiB | 1.3% done | ETA(epoch): 689s + [Epoch 6][40/66] loss=0.940795 avg=0.895579 VRAM=38.8GiB | 1.3% done | ETA(epoch): 498s + [Epoch 6][50/66] loss=0.883447 avg=0.899723 VRAM=39.0GiB | 1.4% done | ETA(epoch): 306s + [Epoch 6][60/66] loss=0.899043 avg=0.894020 VRAM=38.8GiB | 1.4% done | ETA(epoch): 115s + Train loss: 0.893988 (1263.6s) ETA: 12296min + Val loss: 0.919961 [t_0.0-0.2=1.0690 t_0.2-0.4=1.0148 t_0.4-0.6=0.8417 t_0.6-0.8=0.7589 t_0.8-1.0=0.9291] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0006 +[MEM @ epoch 6 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 7/499 (1% done) --- + [Epoch 7][10/66] loss=0.945812 avg=0.919197 VRAM=39.0GiB | 1.4% done | ETA(epoch): 1071s + [Epoch 7][20/66] loss=0.824102 avg=0.896178 VRAM=38.8GiB | 1.5% done | ETA(epoch): 880s + [Epoch 7][30/66] loss=0.857899 avg=0.898495 VRAM=39.0GiB | 1.5% done | ETA(epoch): 689s + [Epoch 7][40/66] loss=0.860950 avg=0.895107 VRAM=38.8GiB | 1.5% done | ETA(epoch): 497s + [Epoch 7][50/66] loss=0.846639 avg=0.887980 VRAM=39.0GiB | 1.6% done | ETA(epoch): 306s + [Epoch 7][60/66] loss=0.900606 avg=0.890836 VRAM=38.8GiB | 1.6% done | ETA(epoch): 115s + Train loss: 0.892324 (1262.3s) ETA: 12128min + Val loss: 0.862610 [t_0.0-0.2=1.0859 t_0.2-0.4=0.9808 t_0.4-0.6=0.8656 t_0.6-0.8=0.7502 t_0.8-1.0=0.7632] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0007 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 8/499 (2% done) --- + [Epoch 8][10/66] loss=0.845001 avg=0.861295 VRAM=39.0GiB | 1.6% done | ETA(epoch): 1072s + [Epoch 8][20/66] loss=0.924297 avg=0.870855 VRAM=38.8GiB | 1.7% done | ETA(epoch): 884s + [Epoch 8][30/66] loss=0.905143 avg=0.872508 VRAM=39.0GiB | 1.7% done | ETA(epoch): 691s + [Epoch 8][40/66] loss=0.926068 avg=0.882110 VRAM=38.8GiB | 1.7% done | ETA(epoch): 499s + [Epoch 8][50/66] loss=0.721362 avg=0.879772 VRAM=39.0GiB | 1.8% done | ETA(epoch): 307s + [Epoch 8][60/66] loss=0.801751 avg=0.879207 VRAM=38.8GiB | 1.8% done | ETA(epoch): 115s + Train loss: 0.876615 (1264.5s) ETA: 11994min + Val loss: 0.879127 [t_0.0-0.2=1.0810 t_0.2-0.4=1.0236 t_0.4-0.6=0.8607 t_0.6-0.8=0.7371 t_0.8-1.0=0.7688] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 16.1/188.5 GiB (8.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 9/499 (2% done) --- + [Epoch 9][10/66] loss=0.928527 avg=0.880146 VRAM=39.0GiB | 1.8% done | ETA(epoch): 1071s + [Epoch 9][20/66] loss=0.834448 avg=0.872435 VRAM=38.8GiB | 1.9% done | ETA(epoch): 880s + [Epoch 9][30/66] loss=0.865812 avg=0.875920 VRAM=39.0GiB | 1.9% done | ETA(epoch): 689s + [Epoch 9][40/66] loss=1.046726 avg=0.878207 VRAM=38.8GiB | 1.9% done | ETA(epoch): 498s + [Epoch 9][50/66] loss=0.743672 avg=0.873505 VRAM=39.0GiB | 2.0% done | ETA(epoch): 306s + [Epoch 9][60/66] loss=0.844119 avg=0.874398 VRAM=38.8GiB | 2.0% done | ETA(epoch): 115s + Train loss: 0.880499 (1264.1s) ETA: 11883min + Val loss: 0.888400 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0221 t_0.4-0.6=0.8586 t_0.6-0.8=0.7197 t_0.8-1.0=0.7422] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 10/499 (2% done) --- + [Epoch 10][10/66] loss=0.820841 avg=0.887861 VRAM=39.0GiB | 2.0% done | ETA(epoch): 1071s + [Epoch 10][20/66] loss=0.813798 avg=0.877345 VRAM=38.8GiB | 2.1% done | ETA(epoch): 880s + [Epoch 10][30/66] loss=0.974298 avg=0.872086 VRAM=39.0GiB | 2.1% done | ETA(epoch): 689s + [Epoch 10][40/66] loss=0.870616 avg=0.876560 VRAM=38.8GiB | 2.1% done | ETA(epoch): 497s + [Epoch 10][50/66] loss=0.848396 avg=0.874737 VRAM=39.0GiB | 2.2% done | ETA(epoch): 306s + [Epoch 10][60/66] loss=0.853350 avg=0.874650 VRAM=38.8GiB | 2.2% done | ETA(epoch): 115s + Train loss: 0.875195 (1264.5s) ETA: 11790min + Val loss: 0.886706 [t_0.0-0.2=1.0771 t_0.2-0.4=1.0191 t_0.4-0.6=0.8451 t_0.6-0.8=0.7239 t_0.8-1.0=0.7737] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0010 +[MEM @ epoch 10 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 11/499 (2% done) --- + [Epoch 11][10/66] loss=0.769477 avg=0.869411 VRAM=39.0GiB | 2.2% done | ETA(epoch): 1071s + [Epoch 11][20/66] loss=0.936199 avg=0.875300 VRAM=38.8GiB | 2.3% done | ETA(epoch): 880s + [Epoch 11][30/66] loss=0.820394 avg=0.873916 VRAM=39.0GiB | 2.3% done | ETA(epoch): 689s + [Epoch 11][40/66] loss=0.815252 avg=0.870492 VRAM=38.8GiB | 2.3% done | ETA(epoch): 497s + [Epoch 11][50/66] loss=0.934335 avg=0.868272 VRAM=39.0GiB | 2.4% done | ETA(epoch): 306s + [Epoch 11][60/66] loss=0.862769 avg=0.865124 VRAM=38.8GiB | 2.4% done | ETA(epoch): 115s + Train loss: 0.864720 (1264.4s) ETA: 11707min + Val loss: 0.882625 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9960 t_0.4-0.6=0.8308 t_0.6-0.8=0.7472 t_0.8-1.0=0.7555] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 12/499 (2% done) --- + [Epoch 12][10/66] loss=0.886075 avg=0.883815 VRAM=39.0GiB | 2.4% done | ETA(epoch): 1071s + [Epoch 12][20/66] loss=0.919132 avg=0.908542 VRAM=38.8GiB | 2.5% done | ETA(epoch): 881s + [Epoch 12][30/66] loss=0.889749 avg=0.900780 VRAM=39.0GiB | 2.5% done | ETA(epoch): 689s + [Epoch 12][40/66] loss=0.896498 avg=0.892164 VRAM=38.8GiB | 2.5% done | ETA(epoch): 498s + [Epoch 12][50/66] loss=0.807658 avg=0.888191 VRAM=39.0GiB | 2.6% done | ETA(epoch): 306s + [Epoch 12][60/66] loss=0.849328 avg=0.887203 VRAM=38.8GiB | 2.6% done | ETA(epoch): 115s + Train loss: 0.884152 (1263.3s) ETA: 11632min + Val loss: 0.877134 [t_0.0-0.2=1.0775 t_0.2-0.4=1.0153 t_0.4-0.6=0.8601 t_0.6-0.8=0.7279 t_0.8-1.0=0.7370] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 13/499 (3% done) --- + [Epoch 13][10/66] loss=0.935314 avg=0.894784 VRAM=39.0GiB | 2.6% done | ETA(epoch): 1071s + [Epoch 13][20/66] loss=0.795491 avg=0.892280 VRAM=38.8GiB | 2.7% done | ETA(epoch): 879s + [Epoch 13][30/66] loss=0.867093 avg=0.896254 VRAM=39.0GiB | 2.7% done | ETA(epoch): 688s + [Epoch 13][40/66] loss=0.926642 avg=0.895744 VRAM=38.8GiB | 2.7% done | ETA(epoch): 498s + [Epoch 13][50/66] loss=0.812497 avg=0.889685 VRAM=39.0GiB | 2.8% done | ETA(epoch): 307s + [Epoch 13][60/66] loss=0.970946 avg=0.886760 VRAM=38.8GiB | 2.8% done | ETA(epoch): 115s + Train loss: 0.885308 (1264.1s) ETA: 11565min + Val loss: 0.864957 [t_0.0-0.2=1.0837 t_0.2-0.4=1.0108 t_0.4-0.6=0.8496 t_0.6-0.8=0.7376 t_0.8-1.0=0.7264] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 14/499 (3% done) --- + [Epoch 14][10/66] loss=0.888987 avg=0.882549 VRAM=39.0GiB | 2.8% done | ETA(epoch): 1072s + [Epoch 14][20/66] loss=0.898243 avg=0.906341 VRAM=38.8GiB | 2.9% done | ETA(epoch): 881s + [Epoch 14][30/66] loss=0.925892 avg=0.896000 VRAM=39.0GiB | 2.9% done | ETA(epoch): 689s + [Epoch 14][40/66] loss=0.917101 avg=0.889051 VRAM=38.8GiB | 2.9% done | ETA(epoch): 498s + [Epoch 14][50/66] loss=0.826155 avg=0.879490 VRAM=39.0GiB | 3.0% done | ETA(epoch): 306s + [Epoch 14][60/66] loss=0.908292 avg=0.879427 VRAM=38.8GiB | 3.0% done | ETA(epoch): 115s + Train loss: 0.878510 (1263.4s) ETA: 11503min + Val loss: 0.873870 [t_0.0-0.2=1.0735 t_0.2-0.4=1.0149 t_0.4-0.6=0.8844 t_0.6-0.8=0.7000 t_0.8-1.0=0.7219] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0014 + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 15/499 (3% done) --- + [Epoch 15][10/66] loss=0.835002 avg=0.869766 VRAM=39.0GiB | 3.0% done | ETA(epoch): 1071s + [Epoch 15][20/66] loss=0.805410 avg=0.874988 VRAM=38.8GiB | 3.1% done | ETA(epoch): 880s + [Epoch 15][30/66] loss=0.849533 avg=0.873193 VRAM=39.0GiB | 3.1% done | ETA(epoch): 693s + [Epoch 15][40/66] loss=0.819443 avg=0.872240 VRAM=38.8GiB | 3.1% done | ETA(epoch): 500s + [Epoch 15][50/66] loss=0.987436 avg=0.875349 VRAM=39.0GiB | 3.2% done | ETA(epoch): 307s + [Epoch 15][60/66] loss=0.847804 avg=0.870160 VRAM=38.8GiB | 3.2% done | ETA(epoch): 115s + Train loss: 0.871074 (1267.4s) ETA: 11449min + Val loss: 0.876153 [t_0.0-0.2=1.0698 t_0.2-0.4=1.0147 t_0.4-0.6=0.8544 t_0.6-0.8=0.6980 t_0.8-1.0=0.7411] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 16/499 (3% done) --- + [Epoch 16][10/66] loss=0.773860 avg=0.890506 VRAM=39.0GiB | 3.2% done | ETA(epoch): 1070s + [Epoch 16][20/66] loss=0.935649 avg=0.887989 VRAM=38.8GiB | 3.3% done | ETA(epoch): 880s + [Epoch 16][30/66] loss=0.927338 avg=0.885840 VRAM=39.0GiB | 3.3% done | ETA(epoch): 688s + [Epoch 16][40/66] loss=0.987658 avg=0.884243 VRAM=38.8GiB | 3.3% done | ETA(epoch): 497s + [Epoch 16][50/66] loss=0.885801 avg=0.886162 VRAM=39.0GiB | 3.4% done | ETA(epoch): 306s + [Epoch 16][60/66] loss=0.797954 avg=0.883605 VRAM=38.8GiB | 3.4% done | ETA(epoch): 115s + Train loss: 0.884113 (1262.3s) ETA: 11396min + Val loss: 0.882723 [t_0.0-0.2=1.0802 t_0.2-0.4=0.9895 t_0.4-0.6=0.8622 t_0.6-0.8=0.7315 t_0.8-1.0=0.7148] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 17/499 (3% done) --- + [Epoch 17][10/66] loss=0.885306 avg=0.860476 VRAM=39.0GiB | 3.4% done | ETA(epoch): 1082s + [Epoch 17][20/66] loss=0.894407 avg=0.860264 VRAM=38.8GiB | 3.5% done | ETA(epoch): 884s + [Epoch 17][30/66] loss=0.859583 avg=0.863457 VRAM=39.0GiB | 3.5% done | ETA(epoch): 691s + [Epoch 17][40/66] loss=0.718257 avg=0.869909 VRAM=38.8GiB | 3.5% done | ETA(epoch): 499s + [Epoch 17][50/66] loss=0.931165 avg=0.867033 VRAM=39.0GiB | 3.6% done | ETA(epoch): 307s + [Epoch 17][60/66] loss=0.998692 avg=0.873003 VRAM=38.8GiB | 3.6% done | ETA(epoch): 115s + Train loss: 0.873129 (1264.4s) ETA: 11347min + Val loss: 0.886121 [t_0.0-0.2=1.0755 t_0.2-0.4=1.0134 t_0.4-0.6=0.8716 t_0.6-0.8=0.7421 t_0.8-1.0=0.7236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 18/499 (4% done) --- + [Epoch 18][10/66] loss=0.861395 avg=0.843632 VRAM=39.0GiB | 3.6% done | ETA(epoch): 1072s + [Epoch 18][20/66] loss=0.836577 avg=0.862150 VRAM=38.8GiB | 3.7% done | ETA(epoch): 880s + [Epoch 18][30/66] loss=0.872706 avg=0.870177 VRAM=39.0GiB | 3.7% done | ETA(epoch): 689s + [Epoch 18][40/66] loss=0.833872 avg=0.866109 VRAM=38.8GiB | 3.7% done | ETA(epoch): 498s + [Epoch 18][50/66] loss=0.856256 avg=0.861549 VRAM=39.0GiB | 3.8% done | ETA(epoch): 306s + [Epoch 18][60/66] loss=0.905643 avg=0.855805 VRAM=38.8GiB | 3.8% done | ETA(epoch): 115s + Train loss: 0.857104 (1265.7s) ETA: 11301min + Val loss: 0.907800 [t_0.0-0.2=1.0844 t_0.2-0.4=1.0435 t_0.4-0.6=0.8527 t_0.6-0.8=0.7205 t_0.8-1.0=0.6953] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 19/499 (4% done) --- + [Epoch 19][10/66] loss=0.936056 avg=0.876013 VRAM=39.0GiB | 3.8% done | ETA(epoch): 1071s + [Epoch 19][20/66] loss=0.846451 avg=0.883054 VRAM=38.8GiB | 3.9% done | ETA(epoch): 880s + [Epoch 19][30/66] loss=0.961462 avg=0.884518 VRAM=39.0GiB | 3.9% done | ETA(epoch): 688s + [Epoch 19][40/66] loss=0.932851 avg=0.889316 VRAM=38.8GiB | 3.9% done | ETA(epoch): 497s + [Epoch 19][50/66] loss=0.870814 avg=0.887712 VRAM=39.0GiB | 4.0% done | ETA(epoch): 306s + [Epoch 19][60/66] loss=0.871505 avg=0.882439 VRAM=38.8GiB | 4.0% done | ETA(epoch): 115s + Train loss: 0.885273 (1266.5s) ETA: 11258min + Val loss: 0.885752 [t_0.0-0.2=1.0615 t_0.2-0.4=1.0213 t_0.4-0.6=0.8563 t_0.6-0.8=0.7470 t_0.8-1.0=0.6989] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 16.1/188.5 GiB (8.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 20/499 (4% done) --- + [Epoch 20][10/66] loss=0.787963 avg=0.847587 VRAM=39.0GiB | 4.0% done | ETA(epoch): 1071s + [Epoch 20][20/66] loss=0.951877 avg=0.855662 VRAM=38.8GiB | 4.1% done | ETA(epoch): 880s + [Epoch 20][30/66] loss=0.887752 avg=0.857911 VRAM=39.0GiB | 4.1% done | ETA(epoch): 689s + [Epoch 20][40/66] loss=0.760774 avg=0.855445 VRAM=38.8GiB | 4.1% done | ETA(epoch): 498s + [Epoch 20][50/66] loss=0.876037 avg=0.865106 VRAM=39.0GiB | 4.2% done | ETA(epoch): 307s + [Epoch 20][60/66] loss=0.819870 avg=0.864722 VRAM=38.8GiB | 4.2% done | ETA(epoch): 115s + Train loss: 0.865728 (1265.5s) ETA: 11217min + Val loss: 0.884420 [t_0.0-0.2=1.0783 t_0.2-0.4=1.0062 t_0.4-0.6=0.8508 t_0.6-0.8=0.7512 t_0.8-1.0=0.7015] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 21/499 (4% done) --- + [Epoch 21][10/66] loss=0.808429 avg=0.858292 VRAM=39.0GiB | 4.2% done | ETA(epoch): 1071s + [Epoch 21][20/66] loss=0.822795 avg=0.845044 VRAM=38.8GiB | 4.3% done | ETA(epoch): 880s + [Epoch 21][30/66] loss=0.842559 avg=0.852083 VRAM=39.0GiB | 4.3% done | ETA(epoch): 689s + [Epoch 21][40/66] loss=0.758970 avg=0.845606 VRAM=38.8GiB | 4.3% done | ETA(epoch): 498s + [Epoch 21][50/66] loss=0.822230 avg=0.850220 VRAM=39.0GiB | 4.4% done | ETA(epoch): 306s + [Epoch 21][60/66] loss=0.910191 avg=0.851562 VRAM=38.8GiB | 4.4% done | ETA(epoch): 115s + Train loss: 0.849782 (1263.4s) ETA: 11176min + Val loss: 0.875018 [t_0.0-0.2=1.0641 t_0.2-0.4=0.9842 t_0.4-0.6=0.8720 t_0.6-0.8=0.7182 t_0.8-1.0=0.6916] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 16.1/188.5 GiB (8.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 22/499 (4% done) --- + [Epoch 22][10/66] loss=0.825677 avg=0.828305 VRAM=39.0GiB | 4.4% done | ETA(epoch): 1072s + [Epoch 22][20/66] loss=0.938172 avg=0.858865 VRAM=38.8GiB | 4.5% done | ETA(epoch): 881s + [Epoch 22][30/66] loss=0.789813 avg=0.848219 VRAM=39.0GiB | 4.5% done | ETA(epoch): 689s + [Epoch 22][40/66] loss=0.874797 avg=0.856413 VRAM=38.8GiB | 4.5% done | ETA(epoch): 498s + [Epoch 22][50/66] loss=0.981055 avg=0.861510 VRAM=39.0GiB | 4.6% done | ETA(epoch): 306s + [Epoch 22][60/66] loss=0.871065 avg=0.860528 VRAM=38.8GiB | 4.6% done | ETA(epoch): 115s + Train loss: 0.864349 (1263.4s) ETA: 11137min + Val loss: 0.877237 [t_0.0-0.2=1.0879 t_0.2-0.4=1.0071 t_0.4-0.6=0.8287 t_0.6-0.8=0.7415 t_0.8-1.0=0.6889] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 23/499 (5% done) --- + [Epoch 23][10/66] loss=0.874227 avg=0.875413 VRAM=39.0GiB | 4.6% done | ETA(epoch): 1071s + [Epoch 23][20/66] loss=0.881190 avg=0.864481 VRAM=38.8GiB | 4.7% done | ETA(epoch): 881s + [Epoch 23][30/66] loss=0.815626 avg=0.858957 VRAM=39.0GiB | 4.7% done | ETA(epoch): 689s + [Epoch 23][40/66] loss=0.829309 avg=0.857657 VRAM=38.8GiB | 4.7% done | ETA(epoch): 498s + [Epoch 23][50/66] loss=0.879701 avg=0.862270 VRAM=39.0GiB | 4.8% done | ETA(epoch): 306s + [Epoch 23][60/66] loss=0.943375 avg=0.864318 VRAM=38.8GiB | 4.8% done | ETA(epoch): 115s + Train loss: 0.870203 (1263.4s) ETA: 11100min + Val loss: 0.897116 [t_0.0-0.2=1.0653 t_0.2-0.4=1.0046 t_0.4-0.6=0.8441 t_0.6-0.8=0.7408 t_0.8-1.0=0.6659] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 24/499 (5% done) --- + [Epoch 24][10/66] loss=0.889272 avg=0.874432 VRAM=39.0GiB | 4.8% done | ETA(epoch): 1071s + [Epoch 24][20/66] loss=0.853659 avg=0.873777 VRAM=38.8GiB | 4.9% done | ETA(epoch): 880s + [Epoch 24][30/66] loss=0.775510 avg=0.861660 VRAM=39.0GiB | 4.9% done | ETA(epoch): 689s + [Epoch 24][40/66] loss=0.788175 avg=0.861960 VRAM=38.8GiB | 4.9% done | ETA(epoch): 497s + [Epoch 24][50/66] loss=0.853578 avg=0.864253 VRAM=39.0GiB | 5.0% done | ETA(epoch): 306s + [Epoch 24][60/66] loss=0.812202 avg=0.868051 VRAM=38.8GiB | 5.0% done | ETA(epoch): 115s + Train loss: 0.867448 (1263.1s) ETA: 11063min + Val loss: 0.885053 [t_0.0-0.2=1.0895 t_0.2-0.4=0.9795 t_0.4-0.6=0.8298 t_0.6-0.8=0.7214 t_0.8-1.0=0.6844] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 16.3/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 25/499 (5% done) --- + [Epoch 25][10/66] loss=0.875362 avg=0.863056 VRAM=39.0GiB | 5.0% done | ETA(epoch): 1071s + [Epoch 25][20/66] loss=0.848353 avg=0.855413 VRAM=38.8GiB | 5.1% done | ETA(epoch): 881s + [Epoch 25][30/66] loss=0.849847 avg=0.861600 VRAM=39.0GiB | 5.1% done | ETA(epoch): 689s + [Epoch 25][40/66] loss=0.860909 avg=0.871254 VRAM=38.8GiB | 5.1% done | ETA(epoch): 498s + [Epoch 25][50/66] loss=0.797136 avg=0.867400 VRAM=39.0GiB | 5.2% done | ETA(epoch): 306s + [Epoch 25][60/66] loss=0.719204 avg=0.859371 VRAM=38.8GiB | 5.2% done | ETA(epoch): 115s + Train loss: 0.855830 (1263.3s) ETA: 11028min + Val loss: 0.871678 [t_0.0-0.2=1.0699 t_0.2-0.4=1.0136 t_0.4-0.6=0.8839 t_0.6-0.8=0.7223 t_0.8-1.0=0.6923] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 26/499 (5% done) --- + [Epoch 26][10/66] loss=0.870632 avg=0.884822 VRAM=39.0GiB | 5.2% done | ETA(epoch): 1071s + [Epoch 26][20/66] loss=0.934637 avg=0.897205 VRAM=38.8GiB | 5.3% done | ETA(epoch): 880s + [Epoch 26][30/66] loss=0.748666 avg=0.872789 VRAM=39.0GiB | 5.3% done | ETA(epoch): 689s + [Epoch 26][40/66] loss=0.845170 avg=0.868354 VRAM=38.8GiB | 5.3% done | ETA(epoch): 497s + [Epoch 26][50/66] loss=0.874739 avg=0.872598 VRAM=39.0GiB | 5.4% done | ETA(epoch): 306s + [Epoch 26][60/66] loss=0.801744 avg=0.869159 VRAM=38.8GiB | 5.4% done | ETA(epoch): 115s + Train loss: 0.864204 (1264.6s) ETA: 10994min + Val loss: 0.871840 [t_0.0-0.2=1.0684 t_0.2-0.4=1.0209 t_0.4-0.6=0.8650 t_0.6-0.8=0.7208 t_0.8-1.0=0.6702] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 27/499 (5% done) --- + [Epoch 27][10/66] loss=0.881987 avg=0.898796 VRAM=39.0GiB | 5.4% done | ETA(epoch): 1071s + [Epoch 27][20/66] loss=0.786324 avg=0.875591 VRAM=38.8GiB | 5.5% done | ETA(epoch): 880s + [Epoch 27][30/66] loss=0.884335 avg=0.869805 VRAM=39.0GiB | 5.5% done | ETA(epoch): 689s + [Epoch 27][40/66] loss=0.902559 avg=0.872600 VRAM=38.8GiB | 5.5% done | ETA(epoch): 498s + [Epoch 27][50/66] loss=0.761036 avg=0.869422 VRAM=39.0GiB | 5.6% done | ETA(epoch): 306s + [Epoch 27][60/66] loss=0.971750 avg=0.867104 VRAM=38.8GiB | 5.6% done | ETA(epoch): 115s + Train loss: 0.868153 (1265.8s) ETA: 10961min + Val loss: 0.862867 [t_0.0-0.2=1.0684 t_0.2-0.4=1.0133 t_0.4-0.6=0.8317 t_0.6-0.8=0.7276 t_0.8-1.0=0.6872] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 28/499 (6% done) --- + [Epoch 28][10/66] loss=1.019171 avg=0.900095 VRAM=39.0GiB | 5.6% done | ETA(epoch): 1071s + [Epoch 28][20/66] loss=0.944710 avg=0.892592 VRAM=38.8GiB | 5.7% done | ETA(epoch): 880s + [Epoch 28][30/66] loss=0.796252 avg=0.874497 VRAM=39.0GiB | 5.7% done | ETA(epoch): 688s + [Epoch 28][40/66] loss=0.932686 avg=0.861609 VRAM=38.8GiB | 5.7% done | ETA(epoch): 497s + [Epoch 28][50/66] loss=0.737747 avg=0.864482 VRAM=39.0GiB | 5.8% done | ETA(epoch): 306s + [Epoch 28][60/66] loss=0.942485 avg=0.874983 VRAM=38.8GiB | 5.8% done | ETA(epoch): 115s + Train loss: 0.877560 (1261.7s) ETA: 10928min + Val loss: 0.854231 [t_0.0-0.2=1.0884 t_0.2-0.4=0.9523 t_0.4-0.6=0.8387 t_0.6-0.8=0.7294 t_0.8-1.0=0.6952] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0028 (BEST) + Deleted old checkpoint: checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 16.1/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 29/499 (6% done) --- + [Epoch 29][10/66] loss=0.876869 avg=0.831906 VRAM=39.0GiB | 5.8% done | ETA(epoch): 1068s + [Epoch 29][20/66] loss=0.794518 avg=0.845142 VRAM=38.8GiB | 5.9% done | ETA(epoch): 879s + [Epoch 29][30/66] loss=0.885471 avg=0.870056 VRAM=39.0GiB | 5.9% done | ETA(epoch): 688s + [Epoch 29][40/66] loss=0.859216 avg=0.863792 VRAM=38.8GiB | 5.9% done | ETA(epoch): 498s + [Epoch 29][50/66] loss=0.899872 avg=0.870523 VRAM=39.0GiB | 6.0% done | ETA(epoch): 306s + [Epoch 29][60/66] loss=0.938829 avg=0.863723 VRAM=38.8GiB | 6.0% done | ETA(epoch): 115s + Train loss: 0.861819 (1264.1s) ETA: 10896min + Val loss: 0.866379 [t_0.0-0.2=1.0769 t_0.2-0.4=0.9876 t_0.4-0.6=0.8561 t_0.6-0.8=0.7586 t_0.8-1.0=0.7004] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 30/499 (6% done) --- + [Epoch 30][10/66] loss=0.780720 avg=0.891705 VRAM=39.0GiB | 6.0% done | ETA(epoch): 1069s + [Epoch 30][20/66] loss=0.834619 avg=0.857466 VRAM=38.8GiB | 6.1% done | ETA(epoch): 878s + [Epoch 30][30/66] loss=0.781763 avg=0.852829 VRAM=39.0GiB | 6.1% done | ETA(epoch): 687s + [Epoch 30][40/66] loss=0.886304 avg=0.852320 VRAM=38.8GiB | 6.1% done | ETA(epoch): 496s + [Epoch 30][50/66] loss=0.840431 avg=0.865754 VRAM=39.0GiB | 6.2% done | ETA(epoch): 305s + [Epoch 30][60/66] loss=0.791126 avg=0.859196 VRAM=38.8GiB | 6.2% done | ETA(epoch): 115s + Train loss: 0.861579 (1260.2s) ETA: 10864min + Val loss: 0.868022 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0108 t_0.4-0.6=0.8403 t_0.6-0.8=0.7269 t_0.8-1.0=0.6770] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 31/499 (6% done) --- + [Epoch 31][10/66] loss=0.822772 avg=0.861569 VRAM=39.0GiB | 6.2% done | ETA(epoch): 1069s + [Epoch 31][20/66] loss=0.742137 avg=0.850894 VRAM=38.8GiB | 6.3% done | ETA(epoch): 882s + [Epoch 31][30/66] loss=0.910846 avg=0.855610 VRAM=39.0GiB | 6.3% done | ETA(epoch): 689s + [Epoch 31][40/66] loss=0.919527 avg=0.863313 VRAM=38.8GiB | 6.3% done | ETA(epoch): 497s + [Epoch 31][50/66] loss=0.979510 avg=0.875586 VRAM=39.0GiB | 6.4% done | ETA(epoch): 306s + [Epoch 31][60/66] loss=0.802112 avg=0.873376 VRAM=38.8GiB | 6.4% done | ETA(epoch): 115s + Train loss: 0.875322 (1261.3s) ETA: 10832min + Val loss: 0.828917 [t_0.0-0.2=1.0808 t_0.2-0.4=1.0320 t_0.4-0.6=0.8275 t_0.6-0.8=0.7117 t_0.8-1.0=0.6927] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0031 (BEST) + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 32/499 (6% done) --- + [Epoch 32][10/66] loss=0.790474 avg=0.850034 VRAM=39.0GiB | 6.4% done | ETA(epoch): 1068s + [Epoch 32][20/66] loss=0.743303 avg=0.843878 VRAM=38.8GiB | 6.5% done | ETA(epoch): 882s + [Epoch 32][30/66] loss=0.989269 avg=0.860817 VRAM=39.0GiB | 6.5% done | ETA(epoch): 690s + [Epoch 32][40/66] loss=0.936098 avg=0.867722 VRAM=38.8GiB | 6.5% done | ETA(epoch): 498s + [Epoch 32][50/66] loss=0.737005 avg=0.860563 VRAM=39.0GiB | 6.6% done | ETA(epoch): 306s + [Epoch 32][60/66] loss=1.017199 avg=0.861351 VRAM=38.8GiB | 6.6% done | ETA(epoch): 115s + Train loss: 0.859653 (1261.9s) ETA: 10801min + Val loss: 0.880593 [t_0.0-0.2=1.0788 t_0.2-0.4=1.0331 t_0.4-0.6=0.8468 t_0.6-0.8=0.7324 t_0.8-1.0=0.6905] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 33/499 (7% done) --- + [Epoch 33][10/66] loss=0.760807 avg=0.867072 VRAM=39.0GiB | 6.6% done | ETA(epoch): 1069s + [Epoch 33][20/66] loss=0.956017 avg=0.842193 VRAM=38.8GiB | 6.7% done | ETA(epoch): 879s + [Epoch 33][30/66] loss=0.872623 avg=0.840261 VRAM=39.0GiB | 6.7% done | ETA(epoch): 688s + [Epoch 33][40/66] loss=0.846239 avg=0.840391 VRAM=38.8GiB | 6.7% done | ETA(epoch): 497s + [Epoch 33][50/66] loss=0.905481 avg=0.851699 VRAM=39.0GiB | 6.8% done | ETA(epoch): 306s + [Epoch 33][60/66] loss=0.764977 avg=0.849880 VRAM=38.8GiB | 6.8% done | ETA(epoch): 115s + Train loss: 0.851903 (1261.0s) ETA: 10771min + Val loss: 0.872860 [t_0.0-0.2=1.0868 t_0.2-0.4=1.0052 t_0.4-0.6=0.8499 t_0.6-0.8=0.7164 t_0.8-1.0=0.6707] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 34/499 (7% done) --- + [Epoch 34][10/66] loss=0.816713 avg=0.855252 VRAM=39.0GiB | 6.8% done | ETA(epoch): 1070s + [Epoch 34][20/66] loss=0.897059 avg=0.856750 VRAM=38.8GiB | 6.9% done | ETA(epoch): 879s + [Epoch 34][30/66] loss=0.855779 avg=0.843377 VRAM=39.0GiB | 6.9% done | ETA(epoch): 688s + [Epoch 34][40/66] loss=0.929487 avg=0.843081 VRAM=38.8GiB | 6.9% done | ETA(epoch): 497s + [Epoch 34][50/66] loss=0.786205 avg=0.855292 VRAM=39.0GiB | 7.0% done | ETA(epoch): 306s + [Epoch 34][60/66] loss=0.814949 avg=0.858973 VRAM=38.8GiB | 7.0% done | ETA(epoch): 115s + Train loss: 0.856656 (1261.4s) ETA: 10741min + Val loss: 0.855071 [t_0.0-0.2=1.0810 t_0.2-0.4=0.9810 t_0.4-0.6=0.8249 t_0.6-0.8=0.7299 t_0.8-1.0=0.6602] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0034 +[MEM @ epoch 34 end] RAM: 17.4/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 35/499 (7% done) --- + [Epoch 35][10/66] loss=0.824940 avg=0.928500 VRAM=39.0GiB | 7.0% done | ETA(epoch): 1074s + [Epoch 35][20/66] loss=0.922620 avg=0.906516 VRAM=38.8GiB | 7.1% done | ETA(epoch): 884s + [Epoch 35][30/66] loss=0.904174 avg=0.893219 VRAM=39.0GiB | 7.1% done | ETA(epoch): 692s + [Epoch 35][40/66] loss=0.935529 avg=0.886677 VRAM=38.8GiB | 7.1% done | ETA(epoch): 499s + [Epoch 35][50/66] loss=0.817416 avg=0.874131 VRAM=39.0GiB | 7.2% done | ETA(epoch): 307s + [Epoch 35][60/66] loss=0.896516 avg=0.874374 VRAM=38.8GiB | 7.2% done | ETA(epoch): 115s + Train loss: 0.872228 (1265.9s) ETA: 10712min + Val loss: 0.851470 [t_0.0-0.2=1.0507 t_0.2-0.4=1.0064 t_0.4-0.6=0.8237 t_0.6-0.8=0.7382 t_0.8-1.0=0.6649] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 17.2/188.5 GiB (9.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 36/499 (7% done) --- + [Epoch 36][10/66] loss=0.763807 avg=0.853137 VRAM=39.0GiB | 7.2% done | ETA(epoch): 1070s + [Epoch 36][20/66] loss=0.934546 avg=0.876624 VRAM=38.8GiB | 7.3% done | ETA(epoch): 879s + [Epoch 36][30/66] loss=0.917519 avg=0.872169 VRAM=39.0GiB | 7.3% done | ETA(epoch): 688s + [Epoch 36][40/66] loss=0.971477 avg=0.872672 VRAM=38.8GiB | 7.3% done | ETA(epoch): 497s + [Epoch 36][50/66] loss=0.946561 avg=0.871503 VRAM=39.0GiB | 7.4% done | ETA(epoch): 306s + [Epoch 36][60/66] loss=0.827615 avg=0.869501 VRAM=38.8GiB | 7.4% done | ETA(epoch): 115s + Train loss: 0.868861 (1260.4s) ETA: 10683min + Val loss: 0.851203 [t_0.0-0.2=1.0714 t_0.2-0.4=1.0291 t_0.4-0.6=0.8855 t_0.6-0.8=0.7056 t_0.8-1.0=0.6670] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 37/499 (7% done) --- + [Epoch 37][10/66] loss=0.846416 avg=0.874083 VRAM=39.0GiB | 7.4% done | ETA(epoch): 1068s + [Epoch 37][20/66] loss=0.913408 avg=0.891302 VRAM=38.8GiB | 7.5% done | ETA(epoch): 877s + [Epoch 37][30/66] loss=0.925050 avg=0.893465 VRAM=39.0GiB | 7.5% done | ETA(epoch): 687s + [Epoch 37][40/66] loss=0.924096 avg=0.879735 VRAM=38.8GiB | 7.5% done | ETA(epoch): 508s + [Epoch 37][50/66] loss=0.915857 avg=0.872851 VRAM=39.0GiB | 7.6% done | ETA(epoch): 311s + [Epoch 37][60/66] loss=0.941961 avg=0.875241 VRAM=38.8GiB | 7.6% done | ETA(epoch): 116s + Train loss: 0.873222 (1277.6s) ETA: 10657min + Val loss: 0.863573 [t_0.0-0.2=1.0801 t_0.2-0.4=0.9743 t_0.4-0.6=0.8578 t_0.6-0.8=0.7312 t_0.8-1.0=0.6896] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 38/499 (8% done) --- + [Epoch 38][10/66] loss=0.819804 avg=0.874146 VRAM=39.0GiB | 7.6% done | ETA(epoch): 1069s + [Epoch 38][20/66] loss=0.780206 avg=0.853159 VRAM=38.8GiB | 7.7% done | ETA(epoch): 878s + [Epoch 38][30/66] loss=0.960263 avg=0.861826 VRAM=39.0GiB | 7.7% done | ETA(epoch): 690s + [Epoch 38][40/66] loss=0.952048 avg=0.864547 VRAM=38.8GiB | 7.7% done | ETA(epoch): 498s + [Epoch 38][50/66] loss=1.041423 avg=0.867806 VRAM=39.0GiB | 7.8% done | ETA(epoch): 306s + [Epoch 38][60/66] loss=0.959938 avg=0.866882 VRAM=38.8GiB | 7.8% done | ETA(epoch): 115s + Train loss: 0.862701 (1262.3s) ETA: 10629min + Val loss: 0.852566 [t_0.0-0.2=1.0934 t_0.2-0.4=0.9929 t_0.4-0.6=0.8438 t_0.6-0.8=0.7289 t_0.8-1.0=0.6777] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 39/499 (8% done) --- + [Epoch 39][10/66] loss=0.915149 avg=0.877545 VRAM=39.0GiB | 7.8% done | ETA(epoch): 1068s + [Epoch 39][20/66] loss=0.734530 avg=0.874929 VRAM=38.8GiB | 7.9% done | ETA(epoch): 885s + [Epoch 39][30/66] loss=0.997476 avg=0.869522 VRAM=39.0GiB | 7.9% done | ETA(epoch): 691s + [Epoch 39][40/66] loss=0.839733 avg=0.864958 VRAM=38.8GiB | 7.9% done | ETA(epoch): 498s + [Epoch 39][50/66] loss=0.961883 avg=0.867145 VRAM=39.0GiB | 8.0% done | ETA(epoch): 306s + [Epoch 39][60/66] loss=0.924932 avg=0.867007 VRAM=38.8GiB | 8.0% done | ETA(epoch): 115s + Train loss: 0.864124 (1262.4s) ETA: 10601min + Val loss: 0.861838 [t_0.0-0.2=1.0887 t_0.2-0.4=0.9749 t_0.4-0.6=0.8661 t_0.6-0.8=0.7019 t_0.8-1.0=0.6696] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 40/499 (8% done) --- + [Epoch 40][10/66] loss=0.895005 avg=0.876613 VRAM=39.0GiB | 8.0% done | ETA(epoch): 1068s + [Epoch 40][20/66] loss=0.823711 avg=0.868805 VRAM=38.8GiB | 8.1% done | ETA(epoch): 877s + [Epoch 40][30/66] loss=0.838376 avg=0.870810 VRAM=39.0GiB | 8.1% done | ETA(epoch): 687s + [Epoch 40][40/66] loss=1.012366 avg=0.870252 VRAM=38.8GiB | 8.1% done | ETA(epoch): 496s + [Epoch 40][50/66] loss=0.977117 avg=0.875349 VRAM=39.0GiB | 8.2% done | ETA(epoch): 305s + [Epoch 40][60/66] loss=0.847001 avg=0.871289 VRAM=38.8GiB | 8.2% done | ETA(epoch): 115s + Train loss: 0.867548 (1259.7s) ETA: 10572min + Val loss: 0.897370 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9794 t_0.4-0.6=0.8515 t_0.6-0.8=0.7252 t_0.8-1.0=0.6992] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 16.4/188.5 GiB (8.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 41/499 (8% done) --- + [Epoch 41][10/66] loss=0.992604 avg=0.875803 VRAM=39.0GiB | 8.2% done | ETA(epoch): 1069s + [Epoch 41][20/66] loss=0.980769 avg=0.896247 VRAM=38.8GiB | 8.3% done | ETA(epoch): 878s + [Epoch 41][30/66] loss=0.800908 avg=0.888875 VRAM=39.0GiB | 8.3% done | ETA(epoch): 687s + [Epoch 41][40/66] loss=0.832435 avg=0.879680 VRAM=38.8GiB | 8.3% done | ETA(epoch): 496s + [Epoch 41][50/66] loss=0.830968 avg=0.875609 VRAM=39.0GiB | 8.4% done | ETA(epoch): 305s + [Epoch 41][60/66] loss=0.814581 avg=0.877623 VRAM=38.8GiB | 8.4% done | ETA(epoch): 115s + Train loss: 0.879669 (1260.2s) ETA: 10544min + Val loss: 0.847661 [t_0.0-0.2=1.0654 t_0.2-0.4=0.9904 t_0.4-0.6=0.8304 t_0.6-0.8=0.7346 t_0.8-1.0=0.6885] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 16.2/188.5 GiB (8.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 42/499 (8% done) --- + [Epoch 42][10/66] loss=0.789884 avg=0.828809 VRAM=39.0GiB | 8.4% done | ETA(epoch): 1070s + [Epoch 42][20/66] loss=0.948202 avg=0.843425 VRAM=38.8GiB | 8.5% done | ETA(epoch): 879s + [Epoch 42][30/66] loss=0.868117 avg=0.846614 VRAM=39.0GiB | 8.5% done | ETA(epoch): 688s + [Epoch 42][40/66] loss=0.820427 avg=0.844120 VRAM=38.8GiB | 8.5% done | ETA(epoch): 497s + [Epoch 42][50/66] loss=0.750398 avg=0.842043 VRAM=39.0GiB | 8.6% done | ETA(epoch): 306s + [Epoch 42][60/66] loss=0.859535 avg=0.846981 VRAM=38.8GiB | 8.6% done | ETA(epoch): 115s + Train loss: 0.849393 (1263.1s) ETA: 10517min + Val loss: 0.861461 [t_0.0-0.2=1.0830 t_0.2-0.4=0.9613 t_0.4-0.6=0.8720 t_0.6-0.8=0.7256 t_0.8-1.0=0.6837] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 43/499 (9% done) --- + [Epoch 43][10/66] loss=0.862585 avg=0.809040 VRAM=39.0GiB | 8.6% done | ETA(epoch): 1069s + [Epoch 43][20/66] loss=0.796383 avg=0.841037 VRAM=38.8GiB | 8.7% done | ETA(epoch): 878s + [Epoch 43][30/66] loss=0.814297 avg=0.844060 VRAM=39.0GiB | 8.7% done | ETA(epoch): 687s + [Epoch 43][40/66] loss=0.783859 avg=0.847849 VRAM=38.8GiB | 8.7% done | ETA(epoch): 496s + [Epoch 43][50/66] loss=0.956222 avg=0.852116 VRAM=39.0GiB | 8.8% done | ETA(epoch): 306s + [Epoch 43][60/66] loss=0.806879 avg=0.854214 VRAM=38.8GiB | 8.8% done | ETA(epoch): 115s + Train loss: 0.853001 (1260.2s) ETA: 10489min + Val loss: 0.850108 [t_0.0-0.2=1.0843 t_0.2-0.4=1.0041 t_0.4-0.6=0.8364 t_0.6-0.8=0.7349 t_0.8-1.0=0.6656] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 44/499 (9% done) --- + [Epoch 44][10/66] loss=0.889040 avg=0.840406 VRAM=39.0GiB | 8.8% done | ETA(epoch): 1068s + [Epoch 44][20/66] loss=0.865110 avg=0.841975 VRAM=38.8GiB | 8.9% done | ETA(epoch): 879s + [Epoch 44][30/66] loss=0.739255 avg=0.839724 VRAM=39.0GiB | 8.9% done | ETA(epoch): 688s + [Epoch 44][40/66] loss=0.736071 avg=0.833992 VRAM=38.8GiB | 8.9% done | ETA(epoch): 497s + [Epoch 44][50/66] loss=0.864688 avg=0.838172 VRAM=39.0GiB | 9.0% done | ETA(epoch): 306s + [Epoch 44][60/66] loss=0.998152 avg=0.847997 VRAM=38.8GiB | 9.0% done | ETA(epoch): 115s + Train loss: 0.850607 (1260.5s) ETA: 10462min + Val loss: 0.848041 [t_0.0-0.2=1.0648 t_0.2-0.4=1.0423 t_0.4-0.6=0.8679 t_0.6-0.8=0.7210 t_0.8-1.0=0.6542] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 45/499 (9% done) --- + [Epoch 45][10/66] loss=0.801797 avg=0.889522 VRAM=39.0GiB | 9.0% done | ETA(epoch): 1068s + [Epoch 45][20/66] loss=0.798412 avg=0.877350 VRAM=38.8GiB | 9.1% done | ETA(epoch): 878s + [Epoch 45][30/66] loss=0.836200 avg=0.868516 VRAM=39.0GiB | 9.1% done | ETA(epoch): 687s + [Epoch 45][40/66] loss=0.957268 avg=0.873619 VRAM=38.8GiB | 9.1% done | ETA(epoch): 498s + [Epoch 45][50/66] loss=0.804535 avg=0.874399 VRAM=39.0GiB | 9.2% done | ETA(epoch): 306s + [Epoch 45][60/66] loss=0.939651 avg=0.872886 VRAM=38.8GiB | 9.2% done | ETA(epoch): 115s + Train loss: 0.870925 (1263.6s) ETA: 10436min + Val loss: 0.827523 [t_0.0-0.2=1.0488 t_0.2-0.4=1.0211 t_0.4-0.6=0.8698 t_0.6-0.8=0.7088 t_0.8-1.0=0.6645] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0045 (BEST) + Deleted old checkpoint: checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 17.6/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 46/499 (9% done) --- + [Epoch 46][10/66] loss=0.931708 avg=0.872054 VRAM=39.0GiB | 9.2% done | ETA(epoch): 1073s + [Epoch 46][20/66] loss=0.862294 avg=0.857503 VRAM=38.8GiB | 9.3% done | ETA(epoch): 881s + [Epoch 46][30/66] loss=0.784662 avg=0.859292 VRAM=39.0GiB | 9.3% done | ETA(epoch): 692s + [Epoch 46][40/66] loss=0.881821 avg=0.856143 VRAM=38.8GiB | 9.3% done | ETA(epoch): 499s + [Epoch 46][50/66] loss=0.798922 avg=0.855479 VRAM=39.0GiB | 9.4% done | ETA(epoch): 307s + [Epoch 46][60/66] loss=0.887376 avg=0.857706 VRAM=38.8GiB | 9.4% done | ETA(epoch): 115s + Train loss: 0.854246 (1267.1s) ETA: 10410min + Val loss: 0.857945 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0178 t_0.4-0.6=0.8740 t_0.6-0.8=0.7268 t_0.8-1.0=0.6817] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 47/499 (9% done) --- + [Epoch 47][10/66] loss=0.873295 avg=0.870595 VRAM=39.0GiB | 9.4% done | ETA(epoch): 1073s + [Epoch 47][20/66] loss=0.706372 avg=0.848916 VRAM=38.8GiB | 9.5% done | ETA(epoch): 886s + [Epoch 47][30/66] loss=0.886073 avg=0.853677 VRAM=39.0GiB | 9.5% done | ETA(epoch): 692s + [Epoch 47][40/66] loss=0.970366 avg=0.855452 VRAM=38.8GiB | 9.5% done | ETA(epoch): 499s + [Epoch 47][50/66] loss=0.824682 avg=0.859154 VRAM=39.0GiB | 9.6% done | ETA(epoch): 307s + [Epoch 47][60/66] loss=0.960196 avg=0.862168 VRAM=38.8GiB | 9.6% done | ETA(epoch): 115s + Train loss: 0.862997 (1266.3s) ETA: 10384min + Val loss: 0.853540 [t_0.0-0.2=1.0949 t_0.2-0.4=0.9872 t_0.4-0.6=0.8479 t_0.6-0.8=0.7054 t_0.8-1.0=0.6808] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 48/499 (10% done) --- + [Epoch 48][10/66] loss=1.009247 avg=0.864932 VRAM=39.0GiB | 9.6% done | ETA(epoch): 1073s + [Epoch 48][20/66] loss=0.798284 avg=0.868536 VRAM=38.8GiB | 9.7% done | ETA(epoch): 882s + [Epoch 48][30/66] loss=0.980788 avg=0.868087 VRAM=39.0GiB | 9.7% done | ETA(epoch): 690s + [Epoch 48][40/66] loss=0.985038 avg=0.872365 VRAM=38.8GiB | 9.7% done | ETA(epoch): 498s + [Epoch 48][50/66] loss=1.011543 avg=0.876512 VRAM=39.0GiB | 9.8% done | ETA(epoch): 307s + [Epoch 48][60/66] loss=0.915721 avg=0.878353 VRAM=38.8GiB | 9.8% done | ETA(epoch): 115s + Train loss: 0.875998 (1264.8s) ETA: 10358min + Val loss: 0.843156 [t_0.0-0.2=1.0652 t_0.2-0.4=1.0009 t_0.4-0.6=0.8406 t_0.6-0.8=0.7160 t_0.8-1.0=0.6726] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0048 +[MEM @ epoch 48 end] RAM: 17.8/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 49/499 (10% done) --- + [Epoch 49][10/66] loss=0.870314 avg=0.894138 VRAM=39.0GiB | 9.8% done | ETA(epoch): 1090s + [Epoch 49][20/66] loss=0.792892 avg=0.880764 VRAM=38.8GiB | 9.9% done | ETA(epoch): 890s + [Epoch 49][30/66] loss=0.802301 avg=0.861920 VRAM=39.0GiB | 9.9% done | ETA(epoch): 696s + [Epoch 49][40/66] loss=0.865949 avg=0.864116 VRAM=38.8GiB | 9.9% done | ETA(epoch): 502s + [Epoch 49][50/66] loss=0.866877 avg=0.873378 VRAM=39.0GiB | 10.0% done | ETA(epoch): 309s + [Epoch 49][60/66] loss=0.972561 avg=0.880614 VRAM=38.8GiB | 10.0% done | ETA(epoch): 116s + Train loss: 0.879057 (1272.7s) ETA: 10334min + Val loss: 0.848242 [t_0.0-0.2=1.0651 t_0.2-0.4=1.0044 t_0.4-0.6=0.8500 t_0.6-0.8=0.7022 t_0.8-1.0=0.6760] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 50/499 (10% done) --- + [MilestoneVis] train_0 step 3300 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3300 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3300 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3300 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3300 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3300 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3300 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3300 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3300 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3300 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 50][10/66] loss=0.719460 avg=0.839227 VRAM=39.0GiB | 10.0% done | ETA(epoch): 6804s + [Epoch 50][20/66] loss=0.786088 avg=0.846602 VRAM=38.8GiB | 10.1% done | ETA(epoch): 3236s + [Epoch 50][30/66] loss=0.768648 avg=0.849443 VRAM=39.0GiB | 10.1% done | ETA(epoch): 1919s + [Epoch 50][40/66] loss=0.945295 avg=0.856880 VRAM=38.8GiB | 10.1% done | ETA(epoch): 1164s + [Epoch 50][50/66] loss=0.842516 avg=0.857697 VRAM=39.0GiB | 10.2% done | ETA(epoch): 634s + [Epoch 50][60/66] loss=0.733345 avg=0.854995 VRAM=38.8GiB | 10.2% done | ETA(epoch): 217s + Train loss: 0.860127 (2292.7s) ETA: 10459min + Val loss: 0.830167 [t_0.0-0.2=1.0645 t_0.2-0.4=0.9713 t_0.4-0.6=0.8511 t_0.6-0.8=0.7244 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 51/499 (10% done) --- + [Epoch 51][10/66] loss=0.875851 avg=0.870883 VRAM=39.0GiB | 10.2% done | ETA(epoch): 1073s + [Epoch 51][20/66] loss=0.841827 avg=0.880273 VRAM=38.8GiB | 10.3% done | ETA(epoch): 882s + [Epoch 51][30/66] loss=0.816870 avg=0.857278 VRAM=39.0GiB | 10.3% done | ETA(epoch): 690s + [Epoch 51][40/66] loss=0.955567 avg=0.855895 VRAM=38.8GiB | 10.3% done | ETA(epoch): 498s + [Epoch 51][50/66] loss=0.867074 avg=0.863436 VRAM=39.0GiB | 10.4% done | ETA(epoch): 307s + [Epoch 51][60/66] loss=0.979836 avg=0.866735 VRAM=38.8GiB | 10.4% done | ETA(epoch): 115s + Train loss: 0.869016 (1264.4s) ETA: 10430min + Val loss: 0.844844 [t_0.0-0.2=1.0724 t_0.2-0.4=1.0012 t_0.4-0.6=0.8136 t_0.6-0.8=0.7401 t_0.8-1.0=0.6490] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 52/499 (10% done) --- + [Epoch 52][10/66] loss=0.889591 avg=0.860935 VRAM=39.0GiB | 10.4% done | ETA(epoch): 1073s + [Epoch 52][20/66] loss=0.828666 avg=0.844988 VRAM=38.8GiB | 10.5% done | ETA(epoch): 882s + [Epoch 52][30/66] loss=0.853324 avg=0.834845 VRAM=39.0GiB | 10.5% done | ETA(epoch): 690s + [Epoch 52][40/66] loss=0.885725 avg=0.843946 VRAM=38.8GiB | 10.5% done | ETA(epoch): 498s + [Epoch 52][50/66] loss=0.694576 avg=0.839410 VRAM=39.0GiB | 10.6% done | ETA(epoch): 307s + [Epoch 52][60/66] loss=0.925247 avg=0.844714 VRAM=38.8GiB | 10.6% done | ETA(epoch): 115s + Train loss: 0.846745 (1265.0s) ETA: 10401min + Val loss: 0.884265 [t_0.0-0.2=1.0665 t_0.2-0.4=0.9830 t_0.4-0.6=0.8232 t_0.6-0.8=0.7204 t_0.8-1.0=0.6730] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 53/499 (11% done) --- + [Epoch 53][10/66] loss=0.899684 avg=0.859153 VRAM=39.0GiB | 10.6% done | ETA(epoch): 1072s + [Epoch 53][20/66] loss=0.815470 avg=0.868640 VRAM=38.8GiB | 10.7% done | ETA(epoch): 882s + [Epoch 53][30/66] loss=0.821530 avg=0.859317 VRAM=39.0GiB | 10.7% done | ETA(epoch): 690s + [Epoch 53][40/66] loss=0.945209 avg=0.858689 VRAM=38.8GiB | 10.7% done | ETA(epoch): 498s + [Epoch 53][50/66] loss=0.796167 avg=0.854061 VRAM=39.0GiB | 10.8% done | ETA(epoch): 307s + [Epoch 53][60/66] loss=0.762038 avg=0.843834 VRAM=38.8GiB | 10.8% done | ETA(epoch): 115s + Train loss: 0.844149 (1265.1s) ETA: 10373min + Val loss: 0.842175 [t_0.0-0.2=1.0786 t_0.2-0.4=0.9542 t_0.4-0.6=0.8375 t_0.6-0.8=0.7446 t_0.8-1.0=0.6566] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 18.3/188.5 GiB (9.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 54/499 (11% done) --- + [Epoch 54][10/66] loss=0.786594 avg=0.841631 VRAM=39.0GiB | 10.8% done | ETA(epoch): 1073s + [Epoch 54][20/66] loss=0.811349 avg=0.854479 VRAM=38.8GiB | 10.9% done | ETA(epoch): 881s + [Epoch 54][30/66] loss=0.878536 avg=0.854594 VRAM=39.0GiB | 10.9% done | ETA(epoch): 690s + [Epoch 54][40/66] loss=0.883894 avg=0.850882 VRAM=38.8GiB | 10.9% done | ETA(epoch): 498s + [Epoch 54][50/66] loss=0.931979 avg=0.856203 VRAM=39.0GiB | 11.0% done | ETA(epoch): 307s + [Epoch 54][60/66] loss=0.736154 avg=0.856741 VRAM=38.8GiB | 11.0% done | ETA(epoch): 115s + Train loss: 0.854601 (1265.2s) ETA: 10345min + Val loss: 0.868478 [t_0.0-0.2=1.0572 t_0.2-0.4=1.0143 t_0.4-0.6=0.8652 t_0.6-0.8=0.7289 t_0.8-1.0=0.6629] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 55/499 (11% done) --- + [Epoch 55][10/66] loss=0.880616 avg=0.891214 VRAM=39.0GiB | 11.0% done | ETA(epoch): 1071s + [Epoch 55][20/66] loss=0.820032 avg=0.874886 VRAM=38.8GiB | 11.1% done | ETA(epoch): 880s + [Epoch 55][30/66] loss=0.958907 avg=0.873379 VRAM=39.0GiB | 11.1% done | ETA(epoch): 692s + [Epoch 55][40/66] loss=0.936940 avg=0.870016 VRAM=38.8GiB | 11.1% done | ETA(epoch): 499s + [Epoch 55][50/66] loss=0.929907 avg=0.872937 VRAM=39.0GiB | 11.2% done | ETA(epoch): 307s + [Epoch 55][60/66] loss=0.822202 avg=0.870660 VRAM=38.8GiB | 11.2% done | ETA(epoch): 115s + Train loss: 0.869721 (1266.7s) ETA: 10318min + Val loss: 0.871528 [t_0.0-0.2=1.0715 t_0.2-0.4=1.0220 t_0.4-0.6=0.8503 t_0.6-0.8=0.7175 t_0.8-1.0=0.6628] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 56/499 (11% done) --- + [Epoch 56][10/66] loss=0.963243 avg=0.869769 VRAM=39.0GiB | 11.2% done | ETA(epoch): 1073s + [Epoch 56][20/66] loss=0.951036 avg=0.860447 VRAM=38.8GiB | 11.3% done | ETA(epoch): 882s + [Epoch 56][30/66] loss=0.791976 avg=0.849303 VRAM=39.0GiB | 11.3% done | ETA(epoch): 690s + [Epoch 56][40/66] loss=0.832278 avg=0.865942 VRAM=38.8GiB | 11.3% done | ETA(epoch): 499s + [Epoch 56][50/66] loss=0.715159 avg=0.863993 VRAM=39.0GiB | 11.4% done | ETA(epoch): 307s + [Epoch 56][60/66] loss=0.924183 avg=0.865031 VRAM=38.8GiB | 11.4% done | ETA(epoch): 115s + Train loss: 0.865467 (1266.5s) ETA: 10290min + Val loss: 0.855192 [t_0.0-0.2=1.0553 t_0.2-0.4=1.0050 t_0.4-0.6=0.8322 t_0.6-0.8=0.7362 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 57/499 (11% done) --- + [Epoch 57][10/66] loss=0.786937 avg=0.825446 VRAM=39.0GiB | 11.4% done | ETA(epoch): 1073s + [Epoch 57][20/66] loss=0.988155 avg=0.831816 VRAM=38.8GiB | 11.5% done | ETA(epoch): 882s + [Epoch 57][30/66] loss=0.703228 avg=0.843792 VRAM=39.0GiB | 11.5% done | ETA(epoch): 690s + [Epoch 57][40/66] loss=0.777449 avg=0.841134 VRAM=38.8GiB | 11.5% done | ETA(epoch): 498s + [Epoch 57][50/66] loss=0.744213 avg=0.840558 VRAM=39.0GiB | 11.6% done | ETA(epoch): 307s + [Epoch 57][60/66] loss=0.765377 avg=0.844689 VRAM=38.8GiB | 11.6% done | ETA(epoch): 115s + Train loss: 0.843826 (1265.6s) ETA: 10263min + Val loss: 0.860142 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9888 t_0.4-0.6=0.8450 t_0.6-0.8=0.7319 t_0.8-1.0=0.6531] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 16.5/188.5 GiB (8.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 58/499 (12% done) --- + [Epoch 58][10/66] loss=0.802907 avg=0.865738 VRAM=39.0GiB | 11.6% done | ETA(epoch): 1072s + [Epoch 58][20/66] loss=0.962486 avg=0.867826 VRAM=38.8GiB | 11.7% done | ETA(epoch): 882s + [Epoch 58][30/66] loss=0.913371 avg=0.867992 VRAM=39.0GiB | 11.7% done | ETA(epoch): 691s + [Epoch 58][40/66] loss=0.793218 avg=0.865610 VRAM=38.8GiB | 11.7% done | ETA(epoch): 499s + [Epoch 58][50/66] loss=1.000310 avg=0.869878 VRAM=39.0GiB | 11.8% done | ETA(epoch): 307s + [Epoch 58][60/66] loss=0.804084 avg=0.872696 VRAM=38.8GiB | 11.8% done | ETA(epoch): 115s + Train loss: 0.871332 (1269.6s) ETA: 10236min + Val loss: 0.839203 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9977 t_0.4-0.6=0.8691 t_0.6-0.8=0.7113 t_0.8-1.0=0.6547] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 59/499 (12% done) --- + [Epoch 59][10/66] loss=0.729358 avg=0.875188 VRAM=39.0GiB | 11.8% done | ETA(epoch): 1074s + [Epoch 59][20/66] loss=0.781316 avg=0.861690 VRAM=38.8GiB | 11.9% done | ETA(epoch): 883s + [Epoch 59][30/66] loss=0.928625 avg=0.858783 VRAM=39.0GiB | 11.9% done | ETA(epoch): 691s + [Epoch 59][40/66] loss=0.762885 avg=0.858752 VRAM=38.8GiB | 11.9% done | ETA(epoch): 499s + [Epoch 59][50/66] loss=0.941751 avg=0.858917 VRAM=39.0GiB | 12.0% done | ETA(epoch): 307s + [Epoch 59][60/66] loss=0.911087 avg=0.860095 VRAM=38.8GiB | 12.0% done | ETA(epoch): 115s + Train loss: 0.851977 (1267.0s) ETA: 10209min + Val loss: 0.886479 [t_0.0-0.2=1.0664 t_0.2-0.4=0.9717 t_0.4-0.6=0.8662 t_0.6-0.8=0.7360 t_0.8-1.0=0.6528] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 60/499 (12% done) --- + [Epoch 60][10/66] loss=0.862218 avg=0.865247 VRAM=39.0GiB | 12.0% done | ETA(epoch): 1073s + [Epoch 60][20/66] loss=0.904572 avg=0.861214 VRAM=38.8GiB | 12.1% done | ETA(epoch): 882s + [Epoch 60][30/66] loss=0.813681 avg=0.871100 VRAM=39.0GiB | 12.1% done | ETA(epoch): 690s + [Epoch 60][40/66] loss=0.927090 avg=0.857844 VRAM=38.8GiB | 12.1% done | ETA(epoch): 498s + [Epoch 60][50/66] loss=0.733275 avg=0.854498 VRAM=39.0GiB | 12.2% done | ETA(epoch): 307s + [Epoch 60][60/66] loss=0.851814 avg=0.855008 VRAM=38.8GiB | 12.2% done | ETA(epoch): 115s + Train loss: 0.860034 (1267.4s) ETA: 10182min + Val loss: 0.841326 [t_0.0-0.2=1.0818 t_0.2-0.4=1.0043 t_0.4-0.6=0.8618 t_0.6-0.8=0.7055 t_0.8-1.0=0.6586] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 61/499 (12% done) --- + [Epoch 61][10/66] loss=0.839671 avg=0.837608 VRAM=39.0GiB | 12.2% done | ETA(epoch): 1073s + [Epoch 61][20/66] loss=0.762953 avg=0.850566 VRAM=38.8GiB | 12.3% done | ETA(epoch): 881s + [Epoch 61][30/66] loss=0.934654 avg=0.863881 VRAM=39.0GiB | 12.3% done | ETA(epoch): 690s + [Epoch 61][40/66] loss=0.881470 avg=0.858198 VRAM=38.8GiB | 12.3% done | ETA(epoch): 498s + [Epoch 61][50/66] loss=0.692309 avg=0.856430 VRAM=39.0GiB | 12.4% done | ETA(epoch): 307s + [Epoch 61][60/66] loss=0.918994 avg=0.858387 VRAM=38.8GiB | 12.4% done | ETA(epoch): 115s + Train loss: 0.858654 (1264.9s) ETA: 10155min + Val loss: 0.854329 [t_0.0-0.2=1.0784 t_0.2-0.4=0.9994 t_0.4-0.6=0.8656 t_0.6-0.8=0.7242 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 62/499 (12% done) --- + [Epoch 62][10/66] loss=0.879156 avg=0.875958 VRAM=39.0GiB | 12.4% done | ETA(epoch): 1074s + [Epoch 62][20/66] loss=0.746356 avg=0.862123 VRAM=38.8GiB | 12.5% done | ETA(epoch): 882s + [Epoch 62][30/66] loss=0.902347 avg=0.857633 VRAM=39.0GiB | 12.5% done | ETA(epoch): 690s + [Epoch 62][40/66] loss=0.798996 avg=0.853282 VRAM=38.8GiB | 12.5% done | ETA(epoch): 498s + [Epoch 62][50/66] loss=0.889570 avg=0.857325 VRAM=39.0GiB | 12.6% done | ETA(epoch): 307s + [Epoch 62][60/66] loss=0.769012 avg=0.862045 VRAM=38.8GiB | 12.6% done | ETA(epoch): 115s + Train loss: 0.863355 (1265.5s) ETA: 10128min + Val loss: 0.842665 [t_0.0-0.2=1.0806 t_0.2-0.4=1.0163 t_0.4-0.6=0.8543 t_0.6-0.8=0.7089 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 63/499 (13% done) --- + [Epoch 63][10/66] loss=0.796473 avg=0.845995 VRAM=39.0GiB | 12.6% done | ETA(epoch): 1073s + [Epoch 63][20/66] loss=0.821781 avg=0.840455 VRAM=38.8GiB | 12.7% done | ETA(epoch): 882s + [Epoch 63][30/66] loss=0.787096 avg=0.836198 VRAM=39.0GiB | 12.7% done | ETA(epoch): 690s + [Epoch 63][40/66] loss=0.929704 avg=0.845983 VRAM=38.8GiB | 12.7% done | ETA(epoch): 498s + [Epoch 63][50/66] loss=0.807839 avg=0.848710 VRAM=39.0GiB | 12.8% done | ETA(epoch): 307s + [Epoch 63][60/66] loss=0.693517 avg=0.852430 VRAM=38.8GiB | 12.8% done | ETA(epoch): 115s + Train loss: 0.852268 (1265.2s) ETA: 10101min + Val loss: 0.844960 [t_0.0-0.2=1.0526 t_0.2-0.4=0.9485 t_0.4-0.6=0.8405 t_0.6-0.8=0.7006 t_0.8-1.0=0.6484] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 64/499 (13% done) --- + [Epoch 64][10/66] loss=0.967195 avg=0.868691 VRAM=39.0GiB | 12.8% done | ETA(epoch): 1073s + [Epoch 64][20/66] loss=0.967164 avg=0.878165 VRAM=38.8GiB | 12.9% done | ETA(epoch): 888s + [Epoch 64][30/66] loss=0.794339 avg=0.868253 VRAM=39.0GiB | 12.9% done | ETA(epoch): 693s + [Epoch 64][40/66] loss=0.934039 avg=0.866204 VRAM=38.8GiB | 12.9% done | ETA(epoch): 500s + [Epoch 64][50/66] loss=0.833401 avg=0.860169 VRAM=39.0GiB | 13.0% done | ETA(epoch): 308s + [Epoch 64][60/66] loss=0.767562 avg=0.860805 VRAM=38.8GiB | 13.0% done | ETA(epoch): 115s + Train loss: 0.862297 (1267.6s) ETA: 10075min + Val loss: 0.838027 [t_0.0-0.2=1.0645 t_0.2-0.4=0.9576 t_0.4-0.6=0.8574 t_0.6-0.8=0.7127 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 65/499 (13% done) --- + [Epoch 65][10/66] loss=0.949755 avg=0.869896 VRAM=39.0GiB | 13.0% done | ETA(epoch): 1073s + [Epoch 65][20/66] loss=0.787969 avg=0.874071 VRAM=38.8GiB | 13.1% done | ETA(epoch): 882s + [Epoch 65][30/66] loss=0.805536 avg=0.867647 VRAM=39.0GiB | 13.1% done | ETA(epoch): 690s + [Epoch 65][40/66] loss=0.929070 avg=0.862006 VRAM=38.8GiB | 13.1% done | ETA(epoch): 498s + [Epoch 65][50/66] loss=0.795740 avg=0.858289 VRAM=39.0GiB | 13.2% done | ETA(epoch): 307s + [Epoch 65][60/66] loss=1.005185 avg=0.860296 VRAM=38.8GiB | 13.2% done | ETA(epoch): 115s + Train loss: 0.861890 (1264.6s) ETA: 10049min + Val loss: 0.868250 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9791 t_0.4-0.6=0.8963 t_0.6-0.8=0.7303 t_0.8-1.0=0.6635] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 66/499 (13% done) --- + [Epoch 66][10/66] loss=0.970906 avg=0.822478 VRAM=39.0GiB | 13.2% done | ETA(epoch): 1074s + [Epoch 66][20/66] loss=0.894948 avg=0.842984 VRAM=38.8GiB | 13.3% done | ETA(epoch): 882s + [Epoch 66][30/66] loss=0.876106 avg=0.841951 VRAM=39.0GiB | 13.3% done | ETA(epoch): 690s + [Epoch 66][40/66] loss=0.976342 avg=0.845956 VRAM=38.8GiB | 13.3% done | ETA(epoch): 498s + [Epoch 66][50/66] loss=0.863103 avg=0.851052 VRAM=39.0GiB | 13.4% done | ETA(epoch): 307s + [Epoch 66][60/66] loss=0.991141 avg=0.852481 VRAM=38.8GiB | 13.4% done | ETA(epoch): 115s + Train loss: 0.848161 (1267.2s) ETA: 10023min + Val loss: 0.859925 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9914 t_0.4-0.6=0.8668 t_0.6-0.8=0.7252 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 67/499 (13% done) --- + [Epoch 67][10/66] loss=0.895148 avg=0.876773 VRAM=39.0GiB | 13.4% done | ETA(epoch): 1073s + [Epoch 67][20/66] loss=0.898558 avg=0.851227 VRAM=38.8GiB | 13.5% done | ETA(epoch): 881s + [Epoch 67][30/66] loss=0.871623 avg=0.847696 VRAM=39.0GiB | 13.5% done | ETA(epoch): 690s + [Epoch 67][40/66] loss=0.804163 avg=0.839725 VRAM=38.8GiB | 13.5% done | ETA(epoch): 499s + [Epoch 67][50/66] loss=0.805720 avg=0.841079 VRAM=39.0GiB | 13.6% done | ETA(epoch): 307s + [Epoch 67][60/66] loss=0.902335 avg=0.844610 VRAM=38.8GiB | 13.6% done | ETA(epoch): 115s + Train loss: 0.845832 (1267.9s) ETA: 9997min + Val loss: 0.863283 [t_0.0-0.2=1.0752 t_0.2-0.4=0.9508 t_0.4-0.6=0.8491 t_0.6-0.8=0.7395 t_0.8-1.0=0.6633] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 68/499 (14% done) --- + [Epoch 68][10/66] loss=0.845003 avg=0.880513 VRAM=39.0GiB | 13.6% done | ETA(epoch): 1073s + [Epoch 68][20/66] loss=0.906444 avg=0.878902 VRAM=38.8GiB | 13.7% done | ETA(epoch): 882s + [Epoch 68][30/66] loss=0.886116 avg=0.869232 VRAM=39.0GiB | 13.7% done | ETA(epoch): 690s + [Epoch 68][40/66] loss=0.771948 avg=0.867655 VRAM=38.8GiB | 13.7% done | ETA(epoch): 499s + [Epoch 68][50/66] loss=0.752635 avg=0.865125 VRAM=39.0GiB | 13.8% done | ETA(epoch): 307s + [Epoch 68][60/66] loss=0.905638 avg=0.866156 VRAM=38.8GiB | 13.8% done | ETA(epoch): 115s + Train loss: 0.863792 (1265.9s) ETA: 9971min + Val loss: 0.853172 [t_0.0-0.2=1.0777 t_0.2-0.4=1.0152 t_0.4-0.6=0.8539 t_0.6-0.8=0.7287 t_0.8-1.0=0.6562] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 69/499 (14% done) --- + [Epoch 69][10/66] loss=0.948173 avg=0.871837 VRAM=39.0GiB | 13.8% done | ETA(epoch): 1072s + [Epoch 69][20/66] loss=0.663945 avg=0.837294 VRAM=38.8GiB | 13.9% done | ETA(epoch): 882s + [Epoch 69][30/66] loss=0.849692 avg=0.847790 VRAM=39.0GiB | 13.9% done | ETA(epoch): 690s + [Epoch 69][40/66] loss=0.853630 avg=0.845116 VRAM=38.8GiB | 13.9% done | ETA(epoch): 501s + [Epoch 69][50/66] loss=0.778508 avg=0.844031 VRAM=39.0GiB | 14.0% done | ETA(epoch): 308s + [Epoch 69][60/66] loss=0.877093 avg=0.846992 VRAM=38.8GiB | 14.0% done | ETA(epoch): 115s + Train loss: 0.848035 (1269.0s) ETA: 9945min + Val loss: 0.850788 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9886 t_0.4-0.6=0.8559 t_0.6-0.8=0.7288 t_0.8-1.0=0.6555] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 70/499 (14% done) --- + [Epoch 70][10/66] loss=0.853800 avg=0.861394 VRAM=39.0GiB | 14.0% done | ETA(epoch): 1073s + [Epoch 70][20/66] loss=0.869055 avg=0.852365 VRAM=38.8GiB | 14.1% done | ETA(epoch): 883s + [Epoch 70][30/66] loss=0.748300 avg=0.858077 VRAM=39.0GiB | 14.1% done | ETA(epoch): 691s + [Epoch 70][40/66] loss=0.954783 avg=0.866134 VRAM=38.8GiB | 14.1% done | ETA(epoch): 499s + [Epoch 70][50/66] loss=0.953620 avg=0.864714 VRAM=39.0GiB | 14.2% done | ETA(epoch): 307s + [Epoch 70][60/66] loss=0.892687 avg=0.863184 VRAM=38.8GiB | 14.2% done | ETA(epoch): 115s + Train loss: 0.864644 (1266.1s) ETA: 9919min + Val loss: 0.860400 [t_0.0-0.2=1.0839 t_0.2-0.4=1.0013 t_0.4-0.6=0.8233 t_0.6-0.8=0.7388 t_0.8-1.0=0.6479] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 71/499 (14% done) --- + [Epoch 71][10/66] loss=0.747417 avg=0.850982 VRAM=39.0GiB | 14.2% done | ETA(epoch): 1075s + [Epoch 71][20/66] loss=0.922120 avg=0.860484 VRAM=38.8GiB | 14.3% done | ETA(epoch): 887s + [Epoch 71][30/66] loss=0.886491 avg=0.858877 VRAM=39.0GiB | 14.3% done | ETA(epoch): 693s + [Epoch 71][40/66] loss=0.902900 avg=0.859224 VRAM=38.8GiB | 14.3% done | ETA(epoch): 500s + [Epoch 71][50/66] loss=0.798748 avg=0.861313 VRAM=39.0GiB | 14.4% done | ETA(epoch): 308s + [Epoch 71][60/66] loss=0.885570 avg=0.860509 VRAM=38.8GiB | 14.4% done | ETA(epoch): 115s + Train loss: 0.862444 (1268.4s) ETA: 9894min + Val loss: 0.829378 [t_0.0-0.2=1.0783 t_0.2-0.4=1.0201 t_0.4-0.6=0.8140 t_0.6-0.8=0.7388 t_0.8-1.0=0.6412] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 72/499 (14% done) --- + [Epoch 72][10/66] loss=0.980048 avg=0.879754 VRAM=39.0GiB | 14.4% done | ETA(epoch): 1073s + [Epoch 72][20/66] loss=0.900985 avg=0.857041 VRAM=38.8GiB | 14.5% done | ETA(epoch): 882s + [Epoch 72][30/66] loss=0.950832 avg=0.862138 VRAM=39.0GiB | 14.5% done | ETA(epoch): 690s + [Epoch 72][40/66] loss=0.875472 avg=0.854755 VRAM=38.8GiB | 14.5% done | ETA(epoch): 498s + [Epoch 72][50/66] loss=0.893086 avg=0.854321 VRAM=39.0GiB | 14.6% done | ETA(epoch): 307s + [Epoch 72][60/66] loss=0.955452 avg=0.855575 VRAM=38.8GiB | 14.6% done | ETA(epoch): 115s + Train loss: 0.852610 (1265.5s) ETA: 9868min + Val loss: 0.870574 [t_0.0-0.2=1.0695 t_0.2-0.4=0.9779 t_0.4-0.6=0.8789 t_0.6-0.8=0.7115 t_0.8-1.0=0.6274] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 73/499 (15% done) --- + [Epoch 73][10/66] loss=0.860247 avg=0.853533 VRAM=39.0GiB | 14.6% done | ETA(epoch): 1073s + [Epoch 73][20/66] loss=0.908195 avg=0.830678 VRAM=38.8GiB | 14.7% done | ETA(epoch): 882s + [Epoch 73][30/66] loss=0.995164 avg=0.847230 VRAM=39.0GiB | 14.7% done | ETA(epoch): 690s + [Epoch 73][40/66] loss=0.826150 avg=0.849693 VRAM=38.8GiB | 14.7% done | ETA(epoch): 498s + [Epoch 73][50/66] loss=0.818483 avg=0.856568 VRAM=39.0GiB | 14.8% done | ETA(epoch): 307s + [Epoch 73][60/66] loss=0.775000 avg=0.857429 VRAM=38.8GiB | 14.8% done | ETA(epoch): 115s + Train loss: 0.859080 (1268.2s) ETA: 9843min + Val loss: 0.859586 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9882 t_0.4-0.6=0.8662 t_0.6-0.8=0.7117 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 16.7/188.5 GiB (8.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 74/499 (15% done) --- + [Epoch 74][10/66] loss=0.772676 avg=0.880971 VRAM=39.0GiB | 14.8% done | ETA(epoch): 1071s + [Epoch 74][20/66] loss=0.753584 avg=0.876271 VRAM=38.8GiB | 14.9% done | ETA(epoch): 882s + [Epoch 74][30/66] loss=1.052681 avg=0.881956 VRAM=39.0GiB | 14.9% done | ETA(epoch): 690s + [Epoch 74][40/66] loss=0.829832 avg=0.866335 VRAM=38.8GiB | 14.9% done | ETA(epoch): 498s + [Epoch 74][50/66] loss=0.828763 avg=0.870275 VRAM=39.0GiB | 15.0% done | ETA(epoch): 307s + [Epoch 74][60/66] loss=1.036022 avg=0.876559 VRAM=38.8GiB | 15.0% done | ETA(epoch): 115s + Train loss: 0.874652 (1264.4s) ETA: 9817min + Val loss: 0.849235 [t_0.0-0.2=1.0653 t_0.2-0.4=1.0074 t_0.4-0.6=0.8517 t_0.6-0.8=0.7081 t_0.8-1.0=0.6254] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 75/499 (15% done) --- + [Epoch 75][10/66] loss=0.946545 avg=0.846070 VRAM=39.0GiB | 15.0% done | ETA(epoch): 1073s + [Epoch 75][20/66] loss=0.765831 avg=0.850688 VRAM=38.8GiB | 15.1% done | ETA(epoch): 882s + [Epoch 75][30/66] loss=0.881399 avg=0.843606 VRAM=39.0GiB | 15.1% done | ETA(epoch): 690s + [Epoch 75][40/66] loss=0.751509 avg=0.842243 VRAM=38.8GiB | 15.1% done | ETA(epoch): 498s + [Epoch 75][50/66] loss=0.808428 avg=0.847958 VRAM=39.0GiB | 15.2% done | ETA(epoch): 307s + [Epoch 75][60/66] loss=0.891940 avg=0.842760 VRAM=38.8GiB | 15.2% done | ETA(epoch): 115s + Train loss: 0.843211 (1265.4s) ETA: 9792min + Val loss: 0.834935 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0595 t_0.4-0.6=0.8482 t_0.6-0.8=0.7232 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 76/499 (15% done) --- + [Epoch 76][10/66] loss=0.894809 avg=0.822690 VRAM=39.0GiB | 15.2% done | ETA(epoch): 1073s + [Epoch 76][20/66] loss=0.774679 avg=0.832493 VRAM=38.8GiB | 15.3% done | ETA(epoch): 881s + [Epoch 76][30/66] loss=0.920355 avg=0.820704 VRAM=39.0GiB | 15.3% done | ETA(epoch): 690s + [Epoch 76][40/66] loss=0.804840 avg=0.822535 VRAM=38.8GiB | 15.3% done | ETA(epoch): 499s + [Epoch 76][50/66] loss=0.715027 avg=0.829684 VRAM=39.0GiB | 15.4% done | ETA(epoch): 307s + [Epoch 76][60/66] loss=0.795025 avg=0.834829 VRAM=38.8GiB | 15.4% done | ETA(epoch): 115s + Train loss: 0.840547 (1265.6s) ETA: 9766min + Val loss: 0.870391 [t_0.0-0.2=1.0560 t_0.2-0.4=1.0108 t_0.4-0.6=0.8488 t_0.6-0.8=0.7334 t_0.8-1.0=0.6443] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 77/499 (15% done) --- + [Epoch 77][10/66] loss=0.906330 avg=0.859347 VRAM=39.0GiB | 15.4% done | ETA(epoch): 1073s + [Epoch 77][20/66] loss=0.911088 avg=0.866483 VRAM=38.8GiB | 15.5% done | ETA(epoch): 881s + [Epoch 77][30/66] loss=0.920131 avg=0.852528 VRAM=39.0GiB | 15.5% done | ETA(epoch): 690s + [Epoch 77][40/66] loss=0.753228 avg=0.846191 VRAM=38.8GiB | 15.5% done | ETA(epoch): 500s + [Epoch 77][50/66] loss=0.870586 avg=0.845424 VRAM=39.0GiB | 15.6% done | ETA(epoch): 307s + [Epoch 77][60/66] loss=0.845817 avg=0.847856 VRAM=38.8GiB | 15.6% done | ETA(epoch): 115s + Train loss: 0.847837 (1267.1s) ETA: 9741min + Val loss: 0.831798 [t_0.0-0.2=1.0666 t_0.2-0.4=0.9665 t_0.4-0.6=0.8285 t_0.6-0.8=0.7102 t_0.8-1.0=0.6506] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 78/499 (16% done) --- + [Epoch 78][10/66] loss=0.780277 avg=0.844290 VRAM=39.0GiB | 15.6% done | ETA(epoch): 1071s + [Epoch 78][20/66] loss=0.799364 avg=0.858565 VRAM=38.8GiB | 15.7% done | ETA(epoch): 880s + [Epoch 78][30/66] loss=0.823783 avg=0.849193 VRAM=39.0GiB | 15.7% done | ETA(epoch): 689s + [Epoch 78][40/66] loss=0.801225 avg=0.845068 VRAM=38.8GiB | 15.7% done | ETA(epoch): 498s + [Epoch 78][50/66] loss=1.012938 avg=0.850535 VRAM=39.0GiB | 15.8% done | ETA(epoch): 306s + [Epoch 78][60/66] loss=0.927135 avg=0.855334 VRAM=38.8GiB | 15.8% done | ETA(epoch): 115s + Train loss: 0.856083 (1263.8s) ETA: 9716min + Val loss: 0.843906 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9931 t_0.4-0.6=0.8675 t_0.6-0.8=0.6955 t_0.8-1.0=0.6389] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 79/499 (16% done) --- + [Epoch 79][10/66] loss=0.927860 avg=0.863310 VRAM=39.0GiB | 15.8% done | ETA(epoch): 1071s + [Epoch 79][20/66] loss=0.817383 avg=0.853847 VRAM=38.8GiB | 15.9% done | ETA(epoch): 881s + [Epoch 79][30/66] loss=0.882138 avg=0.854088 VRAM=39.0GiB | 15.9% done | ETA(epoch): 690s + [Epoch 79][40/66] loss=0.828750 avg=0.849672 VRAM=38.8GiB | 15.9% done | ETA(epoch): 498s + [Epoch 79][50/66] loss=0.866755 avg=0.847665 VRAM=39.0GiB | 16.0% done | ETA(epoch): 306s + [Epoch 79][60/66] loss=0.865755 avg=0.847726 VRAM=38.8GiB | 16.0% done | ETA(epoch): 115s + Train loss: 0.844080 (1264.6s) ETA: 9690min + Val loss: 0.892266 [t_0.0-0.2=1.0811 t_0.2-0.4=1.0175 t_0.4-0.6=0.7950 t_0.6-0.8=0.7006 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 80/499 (16% done) --- + [Epoch 80][10/66] loss=0.849318 avg=0.860368 VRAM=39.0GiB | 16.0% done | ETA(epoch): 1074s + [Epoch 80][20/66] loss=0.870582 avg=0.869840 VRAM=38.8GiB | 16.1% done | ETA(epoch): 882s + [Epoch 80][30/66] loss=0.715971 avg=0.851595 VRAM=39.0GiB | 16.1% done | ETA(epoch): 690s + [Epoch 80][40/66] loss=0.758650 avg=0.841438 VRAM=38.8GiB | 16.1% done | ETA(epoch): 498s + [Epoch 80][50/66] loss=0.932981 avg=0.851174 VRAM=39.0GiB | 16.2% done | ETA(epoch): 307s + [Epoch 80][60/66] loss=0.945294 avg=0.848214 VRAM=38.8GiB | 16.2% done | ETA(epoch): 115s + Train loss: 0.846691 (1264.7s) ETA: 9665min + Val loss: 0.869403 [t_0.0-0.2=1.0812 t_0.2-0.4=0.9985 t_0.4-0.6=0.8247 t_0.6-0.8=0.7203 t_0.8-1.0=0.6605] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 81/499 (16% done) --- + [Epoch 81][10/66] loss=1.043559 avg=0.901713 VRAM=39.0GiB | 16.2% done | ETA(epoch): 1091s + [Epoch 81][20/66] loss=0.762076 avg=0.900521 VRAM=38.8GiB | 16.3% done | ETA(epoch): 889s + [Epoch 81][30/66] loss=0.828668 avg=0.877227 VRAM=39.0GiB | 16.3% done | ETA(epoch): 694s + [Epoch 81][40/66] loss=0.817564 avg=0.864733 VRAM=38.8GiB | 16.3% done | ETA(epoch): 500s + [Epoch 81][50/66] loss=0.882472 avg=0.859514 VRAM=39.0GiB | 16.4% done | ETA(epoch): 308s + [Epoch 81][60/66] loss=0.909201 avg=0.862294 VRAM=38.8GiB | 16.4% done | ETA(epoch): 115s + Train loss: 0.862794 (1269.8s) ETA: 9640min + Val loss: 0.876859 [t_0.0-0.2=1.0785 t_0.2-0.4=0.9428 t_0.4-0.6=0.8829 t_0.6-0.8=0.7025 t_0.8-1.0=0.6584] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 82/499 (16% done) --- + [Epoch 82][10/66] loss=0.880101 avg=0.857538 VRAM=39.0GiB | 16.4% done | ETA(epoch): 1071s + [Epoch 82][20/66] loss=0.982354 avg=0.847980 VRAM=38.8GiB | 16.5% done | ETA(epoch): 880s + [Epoch 82][30/66] loss=0.888943 avg=0.865131 VRAM=39.0GiB | 16.5% done | ETA(epoch): 689s + [Epoch 82][40/66] loss=0.779449 avg=0.868069 VRAM=38.8GiB | 16.5% done | ETA(epoch): 498s + [Epoch 82][50/66] loss=0.737173 avg=0.866182 VRAM=39.0GiB | 16.6% done | ETA(epoch): 306s + [Epoch 82][60/66] loss=0.857390 avg=0.857023 VRAM=38.8GiB | 16.6% done | ETA(epoch): 115s + Train loss: 0.861682 (1266.1s) ETA: 9615min + Val loss: 0.847317 [t_0.0-0.2=1.0771 t_0.2-0.4=1.0158 t_0.4-0.6=0.8499 t_0.6-0.8=0.7053 t_0.8-1.0=0.6574] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 83/499 (17% done) --- + [Epoch 83][10/66] loss=0.837477 avg=0.909650 VRAM=39.0GiB | 16.6% done | ETA(epoch): 1071s + [Epoch 83][20/66] loss=1.009325 avg=0.898718 VRAM=38.8GiB | 16.7% done | ETA(epoch): 881s + [Epoch 83][30/66] loss=0.913532 avg=0.893495 VRAM=39.0GiB | 16.7% done | ETA(epoch): 690s + [Epoch 83][40/66] loss=0.852349 avg=0.879972 VRAM=38.8GiB | 16.7% done | ETA(epoch): 498s + [Epoch 83][50/66] loss=0.910955 avg=0.887887 VRAM=39.0GiB | 16.8% done | ETA(epoch): 307s + [Epoch 83][60/66] loss=0.819090 avg=0.880885 VRAM=38.8GiB | 16.8% done | ETA(epoch): 115s + Train loss: 0.881361 (1264.7s) ETA: 9590min + Val loss: 0.839416 [t_0.0-0.2=1.0684 t_0.2-0.4=1.0393 t_0.4-0.6=0.8586 t_0.6-0.8=0.7284 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 84/499 (17% done) --- + [Epoch 84][10/66] loss=0.861724 avg=0.826150 VRAM=39.0GiB | 16.8% done | ETA(epoch): 1073s + [Epoch 84][20/66] loss=0.934102 avg=0.849079 VRAM=38.8GiB | 16.9% done | ETA(epoch): 881s + [Epoch 84][30/66] loss=0.855671 avg=0.859513 VRAM=39.0GiB | 16.9% done | ETA(epoch): 690s + [Epoch 84][40/66] loss=0.796790 avg=0.849787 VRAM=38.8GiB | 16.9% done | ETA(epoch): 498s + [Epoch 84][50/66] loss=0.912502 avg=0.857143 VRAM=39.0GiB | 17.0% done | ETA(epoch): 307s + [Epoch 84][60/66] loss=0.822148 avg=0.856888 VRAM=38.8GiB | 17.0% done | ETA(epoch): 115s + Train loss: 0.857641 (1264.5s) ETA: 9565min + Val loss: 0.865115 [t_0.0-0.2=1.0838 t_0.2-0.4=0.9944 t_0.4-0.6=0.8758 t_0.6-0.8=0.7214 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 85/499 (17% done) --- + [Epoch 85][10/66] loss=0.872746 avg=0.825248 VRAM=39.0GiB | 17.0% done | ETA(epoch): 1073s + [Epoch 85][20/66] loss=0.779421 avg=0.839441 VRAM=38.8GiB | 17.1% done | ETA(epoch): 881s + [Epoch 85][30/66] loss=0.760834 avg=0.833672 VRAM=39.0GiB | 17.1% done | ETA(epoch): 690s + [Epoch 85][40/66] loss=0.888712 avg=0.840667 VRAM=38.8GiB | 17.1% done | ETA(epoch): 498s + [Epoch 85][50/66] loss=0.819330 avg=0.841763 VRAM=39.0GiB | 17.2% done | ETA(epoch): 307s + [Epoch 85][60/66] loss=0.725930 avg=0.839339 VRAM=38.8GiB | 17.2% done | ETA(epoch): 115s + Train loss: 0.839471 (1264.4s) ETA: 9540min + Val loss: 0.872490 [t_0.0-0.2=1.0832 t_0.2-0.4=1.0150 t_0.4-0.6=0.8818 t_0.6-0.8=0.7217 t_0.8-1.0=0.6416] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 16.9/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 86/499 (17% done) --- + [Epoch 86][10/66] loss=0.955599 avg=0.872673 VRAM=39.0GiB | 17.2% done | ETA(epoch): 1070s + [Epoch 86][20/66] loss=0.856518 avg=0.866728 VRAM=38.8GiB | 17.3% done | ETA(epoch): 881s + [Epoch 86][30/66] loss=0.763477 avg=0.857798 VRAM=39.0GiB | 17.3% done | ETA(epoch): 689s + [Epoch 86][40/66] loss=0.806503 avg=0.856956 VRAM=38.8GiB | 17.3% done | ETA(epoch): 498s + [Epoch 86][50/66] loss=0.932936 avg=0.858675 VRAM=39.0GiB | 17.4% done | ETA(epoch): 306s + [Epoch 86][60/66] loss=0.849060 avg=0.856338 VRAM=38.8GiB | 17.4% done | ETA(epoch): 115s + Train loss: 0.859642 (1263.5s) ETA: 9515min + Val loss: 0.806842 [t_0.0-0.2=1.0679 t_0.2-0.4=0.9804 t_0.4-0.6=0.8414 t_0.6-0.8=0.7168 t_0.8-1.0=0.6469] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0086 (BEST) + Deleted old checkpoint: checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 87/499 (17% done) --- + [Epoch 87][10/66] loss=0.994041 avg=0.893105 VRAM=39.0GiB | 17.4% done | ETA(epoch): 1072s + [Epoch 87][20/66] loss=0.813691 avg=0.884851 VRAM=38.8GiB | 17.5% done | ETA(epoch): 881s + [Epoch 87][30/66] loss=0.775843 avg=0.861284 VRAM=39.0GiB | 17.5% done | ETA(epoch): 690s + [Epoch 87][40/66] loss=0.949420 avg=0.861429 VRAM=38.8GiB | 17.5% done | ETA(epoch): 498s + [Epoch 87][50/66] loss=0.901724 avg=0.861232 VRAM=39.0GiB | 17.6% done | ETA(epoch): 307s + [Epoch 87][60/66] loss=0.732477 avg=0.853074 VRAM=38.8GiB | 17.6% done | ETA(epoch): 115s + Train loss: 0.849666 (1264.8s) ETA: 9490min + Val loss: 0.878315 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9971 t_0.4-0.6=0.8383 t_0.6-0.8=0.7467 t_0.8-1.0=0.6291] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 88/499 (18% done) --- + [Epoch 88][10/66] loss=0.893426 avg=0.820358 VRAM=39.0GiB | 17.6% done | ETA(epoch): 1093s + [Epoch 88][20/66] loss=0.869507 avg=0.820051 VRAM=38.8GiB | 17.7% done | ETA(epoch): 890s + [Epoch 88][30/66] loss=0.812101 avg=0.817548 VRAM=39.0GiB | 17.7% done | ETA(epoch): 694s + [Epoch 88][40/66] loss=0.734877 avg=0.826795 VRAM=38.8GiB | 17.7% done | ETA(epoch): 500s + [Epoch 88][50/66] loss=0.807992 avg=0.825994 VRAM=39.0GiB | 17.8% done | ETA(epoch): 308s + [Epoch 88][60/66] loss=0.785838 avg=0.838249 VRAM=38.8GiB | 17.8% done | ETA(epoch): 115s + Train loss: 0.835930 (1267.8s) ETA: 9466min + Val loss: 0.839835 [t_0.0-0.2=1.0811 t_0.2-0.4=1.0097 t_0.4-0.6=0.8444 t_0.6-0.8=0.7020 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 89/499 (18% done) --- + [Epoch 89][10/66] loss=0.788476 avg=0.870187 VRAM=39.0GiB | 17.8% done | ETA(epoch): 1073s + [Epoch 89][20/66] loss=0.880690 avg=0.855128 VRAM=38.8GiB | 17.9% done | ETA(epoch): 881s + [Epoch 89][30/66] loss=0.912812 avg=0.863448 VRAM=39.0GiB | 17.9% done | ETA(epoch): 690s + [Epoch 89][40/66] loss=0.837567 avg=0.855805 VRAM=38.8GiB | 17.9% done | ETA(epoch): 499s + [Epoch 89][50/66] loss=0.741240 avg=0.854787 VRAM=39.0GiB | 18.0% done | ETA(epoch): 307s + [Epoch 89][60/66] loss=0.891126 avg=0.858729 VRAM=38.8GiB | 18.0% done | ETA(epoch): 115s + Train loss: 0.860110 (1264.7s) ETA: 9442min + Val loss: 0.821146 [t_0.0-0.2=1.0652 t_0.2-0.4=1.0013 t_0.4-0.6=0.8578 t_0.6-0.8=0.7085 t_0.8-1.0=0.6430] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0089 +[MEM @ epoch 89 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 90/499 (18% done) --- + [Epoch 90][10/66] loss=0.767201 avg=0.882595 VRAM=39.0GiB | 18.0% done | ETA(epoch): 1076s + [Epoch 90][20/66] loss=0.992022 avg=0.868101 VRAM=38.8GiB | 18.1% done | ETA(epoch): 884s + [Epoch 90][30/66] loss=0.820187 avg=0.854610 VRAM=39.0GiB | 18.1% done | ETA(epoch): 692s + [Epoch 90][40/66] loss=0.843481 avg=0.861115 VRAM=38.8GiB | 18.1% done | ETA(epoch): 500s + [Epoch 90][50/66] loss=0.905995 avg=0.851920 VRAM=39.0GiB | 18.2% done | ETA(epoch): 308s + [Epoch 90][60/66] loss=0.962037 avg=0.851701 VRAM=38.8GiB | 18.2% done | ETA(epoch): 116s + Train loss: 0.851750 (1270.6s) ETA: 9417min + Val loss: 0.872317 [t_0.0-0.2=1.0654 t_0.2-0.4=0.9916 t_0.4-0.6=0.8583 t_0.6-0.8=0.7197 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 16.7/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 91/499 (18% done) --- + [Epoch 91][10/66] loss=0.931442 avg=0.854420 VRAM=39.0GiB | 18.2% done | ETA(epoch): 1072s + [Epoch 91][20/66] loss=0.910484 avg=0.868980 VRAM=38.8GiB | 18.3% done | ETA(epoch): 881s + [Epoch 91][30/66] loss=0.914879 avg=0.867272 VRAM=39.0GiB | 18.3% done | ETA(epoch): 689s + [Epoch 91][40/66] loss=0.755340 avg=0.873583 VRAM=38.8GiB | 18.3% done | ETA(epoch): 498s + [Epoch 91][50/66] loss=0.806706 avg=0.869017 VRAM=39.0GiB | 18.4% done | ETA(epoch): 306s + [Epoch 91][60/66] loss=0.784674 avg=0.863424 VRAM=38.8GiB | 18.4% done | ETA(epoch): 115s + Train loss: 0.863155 (1264.3s) ETA: 9393min + Val loss: 0.869781 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9875 t_0.4-0.6=0.8416 t_0.6-0.8=0.7082 t_0.8-1.0=0.6458] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 92/499 (18% done) --- + [Epoch 92][10/66] loss=0.728685 avg=0.825591 VRAM=39.0GiB | 18.4% done | ETA(epoch): 1073s + [Epoch 92][20/66] loss=0.877185 avg=0.826992 VRAM=38.8GiB | 18.5% done | ETA(epoch): 881s + [Epoch 92][30/66] loss=0.839924 avg=0.831355 VRAM=39.0GiB | 18.5% done | ETA(epoch): 690s + [Epoch 92][40/66] loss=0.764678 avg=0.828171 VRAM=38.8GiB | 18.5% done | ETA(epoch): 500s + [Epoch 92][50/66] loss=0.779587 avg=0.834112 VRAM=39.0GiB | 18.6% done | ETA(epoch): 307s + [Epoch 92][60/66] loss=0.725126 avg=0.839550 VRAM=38.8GiB | 18.6% done | ETA(epoch): 115s + Train loss: 0.835239 (1267.1s) ETA: 9368min + Val loss: 0.861139 [t_0.0-0.2=1.0690 t_0.2-0.4=1.0166 t_0.4-0.6=0.8381 t_0.6-0.8=0.7168 t_0.8-1.0=0.6495] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 93/499 (19% done) --- + [Epoch 93][10/66] loss=0.725620 avg=0.812659 VRAM=39.0GiB | 18.6% done | ETA(epoch): 1072s + [Epoch 93][20/66] loss=0.888828 avg=0.841894 VRAM=38.8GiB | 18.7% done | ETA(epoch): 881s + [Epoch 93][30/66] loss=0.879047 avg=0.833488 VRAM=39.0GiB | 18.7% done | ETA(epoch): 690s + [Epoch 93][40/66] loss=0.879852 avg=0.841945 VRAM=38.8GiB | 18.7% done | ETA(epoch): 498s + [Epoch 93][50/66] loss=0.801060 avg=0.843958 VRAM=39.0GiB | 18.8% done | ETA(epoch): 307s + [Epoch 93][60/66] loss=0.936203 avg=0.849133 VRAM=38.8GiB | 18.8% done | ETA(epoch): 115s + Train loss: 0.850583 (1265.7s) ETA: 9344min + Val loss: 0.869951 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9787 t_0.4-0.6=0.8133 t_0.6-0.8=0.7059 t_0.8-1.0=0.6558] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 17.0/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 94/499 (19% done) --- + [Epoch 94][10/66] loss=0.917978 avg=0.860946 VRAM=39.0GiB | 18.8% done | ETA(epoch): 1073s + [Epoch 94][20/66] loss=0.740525 avg=0.848557 VRAM=38.8GiB | 18.9% done | ETA(epoch): 882s + [Epoch 94][30/66] loss=0.970500 avg=0.847678 VRAM=39.0GiB | 18.9% done | ETA(epoch): 690s + [Epoch 94][40/66] loss=0.956085 avg=0.854833 VRAM=38.8GiB | 18.9% done | ETA(epoch): 499s + [Epoch 94][50/66] loss=0.779538 avg=0.857173 VRAM=39.0GiB | 19.0% done | ETA(epoch): 307s + [Epoch 94][60/66] loss=0.905436 avg=0.857087 VRAM=38.8GiB | 19.0% done | ETA(epoch): 115s + Train loss: 0.854285 (1265.7s) ETA: 9319min + Val loss: 0.862087 [t_0.0-0.2=1.0771 t_0.2-0.4=0.9952 t_0.4-0.6=0.8175 t_0.6-0.8=0.7279 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 95/499 (19% done) --- + [Epoch 95][10/66] loss=0.933061 avg=0.871165 VRAM=39.0GiB | 19.0% done | ETA(epoch): 1073s + [Epoch 95][20/66] loss=0.918988 avg=0.861548 VRAM=38.8GiB | 19.1% done | ETA(epoch): 882s + [Epoch 95][30/66] loss=0.908975 avg=0.850270 VRAM=39.0GiB | 19.1% done | ETA(epoch): 690s + [Epoch 95][40/66] loss=0.894691 avg=0.848172 VRAM=38.8GiB | 19.1% done | ETA(epoch): 498s + [Epoch 95][50/66] loss=0.945523 avg=0.858305 VRAM=39.0GiB | 19.2% done | ETA(epoch): 307s + [Epoch 95][60/66] loss=0.904561 avg=0.853404 VRAM=38.8GiB | 19.2% done | ETA(epoch): 115s + Train loss: 0.855958 (1265.5s) ETA: 9295min + Val loss: 0.849611 [t_0.0-0.2=1.0691 t_0.2-0.4=1.0249 t_0.4-0.6=0.8404 t_0.6-0.8=0.6982 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 96/499 (19% done) --- + [Epoch 96][10/66] loss=0.913535 avg=0.890976 VRAM=39.0GiB | 19.2% done | ETA(epoch): 1074s + [Epoch 96][20/66] loss=0.985489 avg=0.867075 VRAM=38.8GiB | 19.3% done | ETA(epoch): 882s + [Epoch 96][30/66] loss=0.820874 avg=0.863018 VRAM=39.0GiB | 19.3% done | ETA(epoch): 690s + [Epoch 96][40/66] loss=0.789251 avg=0.867797 VRAM=38.8GiB | 19.3% done | ETA(epoch): 499s + [Epoch 96][50/66] loss=0.913988 avg=0.867800 VRAM=39.0GiB | 19.4% done | ETA(epoch): 307s + [Epoch 96][60/66] loss=0.761142 avg=0.870496 VRAM=38.8GiB | 19.4% done | ETA(epoch): 115s + Train loss: 0.868574 (1266.1s) ETA: 9270min + Val loss: 0.841371 [t_0.0-0.2=1.0751 t_0.2-0.4=1.0137 t_0.4-0.6=0.8287 t_0.6-0.8=0.7120 t_0.8-1.0=0.6340] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 97/499 (19% done) --- + [Epoch 97][10/66] loss=0.883969 avg=0.852132 VRAM=39.0GiB | 19.4% done | ETA(epoch): 1074s + [Epoch 97][20/66] loss=0.800212 avg=0.856765 VRAM=38.8GiB | 19.5% done | ETA(epoch): 883s + [Epoch 97][30/66] loss=0.825603 avg=0.861581 VRAM=39.0GiB | 19.5% done | ETA(epoch): 691s + [Epoch 97][40/66] loss=0.805163 avg=0.846183 VRAM=38.8GiB | 19.5% done | ETA(epoch): 499s + [Epoch 97][50/66] loss=0.872801 avg=0.851991 VRAM=39.0GiB | 19.6% done | ETA(epoch): 307s + [Epoch 97][60/66] loss=0.796360 avg=0.846336 VRAM=38.8GiB | 19.6% done | ETA(epoch): 116s + Train loss: 0.844634 (1270.1s) ETA: 9246min + Val loss: 0.850262 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9821 t_0.4-0.6=0.8561 t_0.6-0.8=0.6751 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 98/499 (20% done) --- + [Epoch 98][10/66] loss=0.911934 avg=0.853150 VRAM=39.0GiB | 19.6% done | ETA(epoch): 1073s + [Epoch 98][20/66] loss=0.890259 avg=0.851816 VRAM=38.8GiB | 19.7% done | ETA(epoch): 883s + [Epoch 98][30/66] loss=0.876318 avg=0.851626 VRAM=39.0GiB | 19.7% done | ETA(epoch): 692s + [Epoch 98][40/66] loss=0.812078 avg=0.845302 VRAM=38.8GiB | 19.7% done | ETA(epoch): 499s + [Epoch 98][50/66] loss=0.884823 avg=0.843065 VRAM=39.0GiB | 19.8% done | ETA(epoch): 307s + [Epoch 98][60/66] loss=0.890337 avg=0.835414 VRAM=38.8GiB | 19.8% done | ETA(epoch): 115s + Train loss: 0.836223 (1268.0s) ETA: 9222min + Val loss: 0.829551 [t_0.0-0.2=1.0635 t_0.2-0.4=1.0009 t_0.4-0.6=0.8690 t_0.6-0.8=0.7167 t_0.8-1.0=0.6477] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 16.8/188.5 GiB (8.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 99/499 (20% done) --- + [Epoch 99][10/66] loss=0.784047 avg=0.878712 VRAM=39.0GiB | 19.8% done | ETA(epoch): 1074s + [Epoch 99][20/66] loss=0.907121 avg=0.859271 VRAM=38.8GiB | 19.9% done | ETA(epoch): 883s + [Epoch 99][30/66] loss=0.885310 avg=0.869123 VRAM=39.0GiB | 19.9% done | ETA(epoch): 691s + [Epoch 99][40/66] loss=0.871564 avg=0.867747 VRAM=38.8GiB | 19.9% done | ETA(epoch): 499s + [Epoch 99][50/66] loss=0.842372 avg=0.866135 VRAM=39.0GiB | 20.0% done | ETA(epoch): 307s + [Epoch 99][60/66] loss=1.005921 avg=0.864746 VRAM=38.8GiB | 20.0% done | ETA(epoch): 115s + Train loss: 0.862202 (1267.5s) ETA: 9198min + Val loss: 0.863059 [t_0.0-0.2=1.0874 t_0.2-0.4=0.9961 t_0.4-0.6=0.8588 t_0.6-0.8=0.7076 t_0.8-1.0=0.6495] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 16.9/188.5 GiB (9.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 100/499 (20% done) --- + [MilestoneVis] train_0 step 6600 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 6600 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 6600 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 6600 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 6600 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 6600 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 6600 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 6600 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 6600 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 6600 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 100][10/66] loss=0.804177 avg=0.856927 VRAM=39.0GiB | 20.0% done | ETA(epoch): 6824s + [Epoch 100][20/66] loss=0.744072 avg=0.857355 VRAM=38.8GiB | 20.1% done | ETA(epoch): 3244s + [Epoch 100][30/66] loss=0.755357 avg=0.846704 VRAM=39.0GiB | 20.1% done | ETA(epoch): 1923s + [Epoch 100][40/66] loss=0.708955 avg=0.844488 VRAM=38.8GiB | 20.1% done | ETA(epoch): 1167s + [Epoch 100][50/66] loss=0.902705 avg=0.848367 VRAM=39.0GiB | 20.2% done | ETA(epoch): 636s + [Epoch 100][60/66] loss=0.823421 avg=0.850800 VRAM=38.8GiB | 20.2% done | ETA(epoch): 218s + Train loss: 0.851983 (2294.2s) ETA: 9242min + Val loss: 0.862553 [t_0.0-0.2=1.0790 t_0.2-0.4=0.9769 t_0.4-0.6=0.8370 t_0.6-0.8=0.7307 t_0.8-1.0=0.6532] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 101/499 (20% done) --- + [Epoch 101][10/66] loss=0.846853 avg=0.866246 VRAM=39.0GiB | 20.2% done | ETA(epoch): 1075s + [Epoch 101][20/66] loss=0.799632 avg=0.859920 VRAM=38.8GiB | 20.3% done | ETA(epoch): 883s + [Epoch 101][30/66] loss=0.810104 avg=0.848322 VRAM=39.0GiB | 20.3% done | ETA(epoch): 691s + [Epoch 101][40/66] loss=0.923491 avg=0.850848 VRAM=38.8GiB | 20.3% done | ETA(epoch): 500s + [Epoch 101][50/66] loss=0.950581 avg=0.850864 VRAM=39.0GiB | 20.4% done | ETA(epoch): 307s + [Epoch 101][60/66] loss=0.876423 avg=0.845005 VRAM=38.8GiB | 20.4% done | ETA(epoch): 115s + Train loss: 0.849795 (1268.1s) ETA: 9217min + Val loss: 0.846694 [t_0.0-0.2=1.0906 t_0.2-0.4=0.9929 t_0.4-0.6=0.8508 t_0.6-0.8=0.7073 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 102/499 (20% done) --- + [Epoch 102][10/66] loss=0.759834 avg=0.809658 VRAM=39.0GiB | 20.4% done | ETA(epoch): 1075s + [Epoch 102][20/66] loss=0.869805 avg=0.827221 VRAM=38.8GiB | 20.5% done | ETA(epoch): 884s + [Epoch 102][30/66] loss=0.830819 avg=0.834749 VRAM=39.0GiB | 20.5% done | ETA(epoch): 692s + [Epoch 102][40/66] loss=0.860574 avg=0.837534 VRAM=38.8GiB | 20.5% done | ETA(epoch): 500s + [Epoch 102][50/66] loss=0.863912 avg=0.843274 VRAM=39.0GiB | 20.6% done | ETA(epoch): 307s + [Epoch 102][60/66] loss=0.882129 avg=0.846949 VRAM=38.8GiB | 20.6% done | ETA(epoch): 115s + Train loss: 0.849889 (1267.7s) ETA: 9192min + Val loss: 0.865712 [t_0.0-0.2=1.0695 t_0.2-0.4=0.9921 t_0.4-0.6=0.8350 t_0.6-0.8=0.7037 t_0.8-1.0=0.6417] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 103/499 (21% done) --- + [Epoch 103][10/66] loss=0.923082 avg=0.850624 VRAM=39.0GiB | 20.6% done | ETA(epoch): 1075s + [Epoch 103][20/66] loss=0.849508 avg=0.819490 VRAM=38.8GiB | 20.7% done | ETA(epoch): 883s + [Epoch 103][30/66] loss=0.979253 avg=0.839173 VRAM=39.0GiB | 20.7% done | ETA(epoch): 691s + [Epoch 103][40/66] loss=0.813167 avg=0.841651 VRAM=38.8GiB | 20.7% done | ETA(epoch): 499s + [Epoch 103][50/66] loss=0.774500 avg=0.839317 VRAM=39.0GiB | 20.8% done | ETA(epoch): 307s + [Epoch 103][60/66] loss=0.963701 avg=0.849610 VRAM=38.8GiB | 20.8% done | ETA(epoch): 115s + Train loss: 0.854003 (1267.2s) ETA: 9167min + Val loss: 0.886551 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9781 t_0.4-0.6=0.8616 t_0.6-0.8=0.7167 t_0.8-1.0=0.6611] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 104/499 (21% done) --- + [Epoch 104][10/66] loss=0.795792 avg=0.829151 VRAM=39.0GiB | 20.8% done | ETA(epoch): 1074s + [Epoch 104][20/66] loss=0.871708 avg=0.859061 VRAM=38.8GiB | 20.9% done | ETA(epoch): 883s + [Epoch 104][30/66] loss=0.820258 avg=0.858028 VRAM=39.0GiB | 20.9% done | ETA(epoch): 691s + [Epoch 104][40/66] loss=0.728559 avg=0.847259 VRAM=38.8GiB | 20.9% done | ETA(epoch): 499s + [Epoch 104][50/66] loss=0.895919 avg=0.850745 VRAM=39.0GiB | 21.0% done | ETA(epoch): 307s + [Epoch 104][60/66] loss=0.820086 avg=0.850067 VRAM=38.8GiB | 21.0% done | ETA(epoch): 115s + Train loss: 0.850753 (1266.8s) ETA: 9142min + Val loss: 0.830640 [t_0.0-0.2=1.0906 t_0.2-0.4=1.0108 t_0.4-0.6=0.8402 t_0.6-0.8=0.7162 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 105/499 (21% done) --- + [Epoch 105][10/66] loss=0.751828 avg=0.838322 VRAM=39.0GiB | 21.0% done | ETA(epoch): 1074s + [Epoch 105][20/66] loss=0.808277 avg=0.860778 VRAM=38.8GiB | 21.1% done | ETA(epoch): 882s + [Epoch 105][30/66] loss=0.872522 avg=0.855097 VRAM=39.0GiB | 21.1% done | ETA(epoch): 692s + [Epoch 105][40/66] loss=0.862810 avg=0.848837 VRAM=38.8GiB | 21.1% done | ETA(epoch): 499s + [Epoch 105][50/66] loss=0.872304 avg=0.856555 VRAM=39.0GiB | 21.2% done | ETA(epoch): 307s + [Epoch 105][60/66] loss=0.836101 avg=0.856507 VRAM=38.8GiB | 21.2% done | ETA(epoch): 115s + Train loss: 0.857371 (1267.8s) ETA: 9117min + Val loss: 0.863332 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9600 t_0.4-0.6=0.8436 t_0.6-0.8=0.7361 t_0.8-1.0=0.6302] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 106/499 (21% done) --- + [Epoch 106][10/66] loss=0.758302 avg=0.869181 VRAM=39.0GiB | 21.2% done | ETA(epoch): 1075s + [Epoch 106][20/66] loss=0.938025 avg=0.858943 VRAM=38.8GiB | 21.3% done | ETA(epoch): 884s + [Epoch 106][30/66] loss=0.769409 avg=0.846240 VRAM=39.0GiB | 21.3% done | ETA(epoch): 691s + [Epoch 106][40/66] loss=0.896964 avg=0.848535 VRAM=38.8GiB | 21.3% done | ETA(epoch): 499s + [Epoch 106][50/66] loss=0.979594 avg=0.844459 VRAM=39.0GiB | 21.4% done | ETA(epoch): 307s + [Epoch 106][60/66] loss=0.863085 avg=0.846839 VRAM=38.8GiB | 21.4% done | ETA(epoch): 115s + Train loss: 0.845568 (1268.2s) ETA: 9093min + Val loss: 0.857522 [t_0.0-0.2=1.0640 t_0.2-0.4=0.9845 t_0.4-0.6=0.8453 t_0.6-0.8=0.7237 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 17.3/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 107/499 (21% done) --- + [Epoch 107][10/66] loss=0.760639 avg=0.839193 VRAM=39.0GiB | 21.4% done | ETA(epoch): 1075s + [Epoch 107][20/66] loss=0.818441 avg=0.837542 VRAM=38.8GiB | 21.5% done | ETA(epoch): 883s + [Epoch 107][30/66] loss=0.754853 avg=0.831983 VRAM=39.0GiB | 21.5% done | ETA(epoch): 691s + [Epoch 107][40/66] loss=0.766923 avg=0.842205 VRAM=38.8GiB | 21.5% done | ETA(epoch): 499s + [Epoch 107][50/66] loss=0.760271 avg=0.843066 VRAM=39.0GiB | 21.6% done | ETA(epoch): 307s + [Epoch 107][60/66] loss=0.916370 avg=0.850112 VRAM=38.8GiB | 21.6% done | ETA(epoch): 115s + Train loss: 0.853104 (1267.6s) ETA: 9068min + Val loss: 0.865968 [t_0.0-0.2=1.0801 t_0.2-0.4=1.0055 t_0.4-0.6=0.8301 t_0.6-0.8=0.7178 t_0.8-1.0=0.6568] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 108/499 (22% done) --- + [Epoch 108][10/66] loss=1.001669 avg=0.824921 VRAM=39.0GiB | 21.6% done | ETA(epoch): 1076s + [Epoch 108][20/66] loss=0.832718 avg=0.833758 VRAM=38.8GiB | 21.7% done | ETA(epoch): 883s + [Epoch 108][30/66] loss=0.912673 avg=0.836255 VRAM=39.0GiB | 21.7% done | ETA(epoch): 691s + [Epoch 108][40/66] loss=0.894175 avg=0.844500 VRAM=38.8GiB | 21.7% done | ETA(epoch): 501s + [Epoch 108][50/66] loss=0.781409 avg=0.843846 VRAM=39.0GiB | 21.8% done | ETA(epoch): 308s + [Epoch 108][60/66] loss=0.817904 avg=0.849923 VRAM=38.8GiB | 21.8% done | ETA(epoch): 115s + Train loss: 0.844877 (1270.3s) ETA: 9043min + Val loss: 0.868021 [t_0.0-0.2=1.0862 t_0.2-0.4=0.9785 t_0.4-0.6=0.8700 t_0.6-0.8=0.7265 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 109/499 (22% done) --- + [Epoch 109][10/66] loss=0.899700 avg=0.843878 VRAM=39.0GiB | 21.8% done | ETA(epoch): 1075s + [Epoch 109][20/66] loss=0.727208 avg=0.837162 VRAM=38.8GiB | 21.9% done | ETA(epoch): 884s + [Epoch 109][30/66] loss=0.775491 avg=0.839892 VRAM=39.0GiB | 21.9% done | ETA(epoch): 692s + [Epoch 109][40/66] loss=0.878200 avg=0.846657 VRAM=38.8GiB | 21.9% done | ETA(epoch): 501s + [Epoch 109][50/66] loss=0.763649 avg=0.838279 VRAM=39.0GiB | 22.0% done | ETA(epoch): 308s + [Epoch 109][60/66] loss=0.889554 avg=0.839757 VRAM=38.8GiB | 22.0% done | ETA(epoch): 115s + Train loss: 0.839345 (1270.0s) ETA: 9019min + Val loss: 0.871413 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9835 t_0.4-0.6=0.8378 t_0.6-0.8=0.7214 t_0.8-1.0=0.6486] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 110/499 (22% done) --- + [Epoch 110][10/66] loss=0.807053 avg=0.867101 VRAM=39.0GiB | 22.0% done | ETA(epoch): 1074s + [Epoch 110][20/66] loss=0.828601 avg=0.835962 VRAM=38.8GiB | 22.1% done | ETA(epoch): 883s + [Epoch 110][30/66] loss=0.825012 avg=0.830463 VRAM=39.0GiB | 22.1% done | ETA(epoch): 691s + [Epoch 110][40/66] loss=0.788047 avg=0.835239 VRAM=38.8GiB | 22.1% done | ETA(epoch): 499s + [Epoch 110][50/66] loss=0.908872 avg=0.839098 VRAM=39.0GiB | 22.2% done | ETA(epoch): 307s + [Epoch 110][60/66] loss=0.823280 avg=0.846809 VRAM=38.8GiB | 22.2% done | ETA(epoch): 115s + Train loss: 0.851090 (1267.1s) ETA: 8994min + Val loss: 0.866571 [t_0.0-0.2=1.0709 t_0.2-0.4=1.0150 t_0.4-0.6=0.8492 t_0.6-0.8=0.7129 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 111/499 (22% done) --- + [Epoch 111][10/66] loss=0.916935 avg=0.846175 VRAM=39.0GiB | 22.2% done | ETA(epoch): 1077s + [Epoch 111][20/66] loss=0.899841 avg=0.853912 VRAM=38.8GiB | 22.3% done | ETA(epoch): 884s + [Epoch 111][30/66] loss=0.870258 avg=0.844593 VRAM=39.0GiB | 22.3% done | ETA(epoch): 692s + [Epoch 111][40/66] loss=0.916728 avg=0.852522 VRAM=38.8GiB | 22.3% done | ETA(epoch): 500s + [Epoch 111][50/66] loss=0.844940 avg=0.849425 VRAM=39.0GiB | 22.4% done | ETA(epoch): 307s + [Epoch 111][60/66] loss=0.895121 avg=0.854778 VRAM=38.8GiB | 22.4% done | ETA(epoch): 115s + Train loss: 0.855393 (1268.1s) ETA: 8970min + Val loss: 0.833689 [t_0.0-0.2=1.0793 t_0.2-0.4=0.9795 t_0.4-0.6=0.8726 t_0.6-0.8=0.7036 t_0.8-1.0=0.6273] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 112/499 (22% done) --- + [Epoch 112][10/66] loss=0.852689 avg=0.843412 VRAM=39.0GiB | 22.4% done | ETA(epoch): 1076s + [Epoch 112][20/66] loss=0.825725 avg=0.836413 VRAM=38.8GiB | 22.5% done | ETA(epoch): 884s + [Epoch 112][30/66] loss=0.916488 avg=0.846409 VRAM=39.0GiB | 22.5% done | ETA(epoch): 691s + [Epoch 112][40/66] loss=0.980659 avg=0.853727 VRAM=38.8GiB | 22.5% done | ETA(epoch): 499s + [Epoch 112][50/66] loss=0.960702 avg=0.856868 VRAM=39.0GiB | 22.6% done | ETA(epoch): 307s + [Epoch 112][60/66] loss=0.903907 avg=0.857958 VRAM=38.8GiB | 22.6% done | ETA(epoch): 115s + Train loss: 0.857727 (1268.3s) ETA: 8945min + Val loss: 0.880052 [t_0.0-0.2=1.0669 t_0.2-0.4=1.0300 t_0.4-0.6=0.8231 t_0.6-0.8=0.7446 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 113/499 (23% done) --- + [Epoch 113][10/66] loss=0.848056 avg=0.872621 VRAM=39.0GiB | 22.6% done | ETA(epoch): 1075s + [Epoch 113][20/66] loss=0.717406 avg=0.881346 VRAM=38.8GiB | 22.7% done | ETA(epoch): 883s + [Epoch 113][30/66] loss=0.707656 avg=0.855769 VRAM=39.0GiB | 22.7% done | ETA(epoch): 691s + [Epoch 113][40/66] loss=0.807496 avg=0.858291 VRAM=38.8GiB | 22.7% done | ETA(epoch): 499s + [Epoch 113][50/66] loss=0.750360 avg=0.855010 VRAM=39.0GiB | 22.8% done | ETA(epoch): 307s + [Epoch 113][60/66] loss=0.896762 avg=0.858647 VRAM=38.8GiB | 22.8% done | ETA(epoch): 115s + Train loss: 0.855128 (1268.4s) ETA: 8921min + Val loss: 0.860137 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0100 t_0.4-0.6=0.8422 t_0.6-0.8=0.7046 t_0.8-1.0=0.6434] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 114/499 (23% done) --- + [Epoch 114][10/66] loss=0.930960 avg=0.881486 VRAM=39.0GiB | 22.8% done | ETA(epoch): 1076s + [Epoch 114][20/66] loss=0.844813 avg=0.872790 VRAM=38.8GiB | 22.9% done | ETA(epoch): 883s + [Epoch 114][30/66] loss=0.962173 avg=0.878120 VRAM=39.0GiB | 22.9% done | ETA(epoch): 691s + [Epoch 114][40/66] loss=0.795136 avg=0.859800 VRAM=38.8GiB | 22.9% done | ETA(epoch): 499s + [Epoch 114][50/66] loss=0.858902 avg=0.858791 VRAM=39.0GiB | 23.0% done | ETA(epoch): 307s + [Epoch 114][60/66] loss=0.825698 avg=0.859250 VRAM=38.8GiB | 23.0% done | ETA(epoch): 115s + Train loss: 0.855877 (1267.4s) ETA: 8896min + Val loss: 0.866671 [t_0.0-0.2=1.0680 t_0.2-0.4=0.9534 t_0.4-0.6=0.8809 t_0.6-0.8=0.6993 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 115/499 (23% done) --- + [Epoch 115][10/66] loss=0.786344 avg=0.852282 VRAM=39.0GiB | 23.0% done | ETA(epoch): 1076s + [Epoch 115][20/66] loss=0.809362 avg=0.828471 VRAM=38.8GiB | 23.1% done | ETA(epoch): 884s + [Epoch 115][30/66] loss=0.923146 avg=0.844296 VRAM=39.0GiB | 23.1% done | ETA(epoch): 691s + [Epoch 115][40/66] loss=0.906116 avg=0.842821 VRAM=38.8GiB | 23.1% done | ETA(epoch): 500s + [Epoch 115][50/66] loss=0.823607 avg=0.845164 VRAM=39.0GiB | 23.2% done | ETA(epoch): 307s + [Epoch 115][60/66] loss=0.850641 avg=0.841236 VRAM=38.8GiB | 23.2% done | ETA(epoch): 115s + Train loss: 0.842364 (1268.2s) ETA: 8872min + Val loss: 0.847171 [t_0.0-0.2=1.0549 t_0.2-0.4=1.0206 t_0.4-0.6=0.8374 t_0.6-0.8=0.7115 t_0.8-1.0=0.6445] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 116/499 (23% done) --- + [Epoch 116][10/66] loss=0.764226 avg=0.869692 VRAM=39.0GiB | 23.2% done | ETA(epoch): 1075s + [Epoch 116][20/66] loss=0.865755 avg=0.886541 VRAM=38.8GiB | 23.3% done | ETA(epoch): 882s + [Epoch 116][30/66] loss=0.908956 avg=0.881835 VRAM=39.0GiB | 23.3% done | ETA(epoch): 691s + [Epoch 116][40/66] loss=0.809034 avg=0.877354 VRAM=38.8GiB | 23.3% done | ETA(epoch): 500s + [Epoch 116][50/66] loss=0.820940 avg=0.873276 VRAM=39.0GiB | 23.4% done | ETA(epoch): 308s + [Epoch 116][60/66] loss=0.786705 avg=0.864500 VRAM=38.8GiB | 23.4% done | ETA(epoch): 115s + Train loss: 0.862833 (1269.4s) ETA: 8848min + Val loss: 0.844743 [t_0.0-0.2=1.0375 t_0.2-0.4=0.9974 t_0.4-0.6=0.8480 t_0.6-0.8=0.7180 t_0.8-1.0=0.6314] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 117/499 (23% done) --- + [Epoch 117][10/66] loss=0.902459 avg=0.878398 VRAM=39.0GiB | 23.4% done | ETA(epoch): 1075s + [Epoch 117][20/66] loss=0.857234 avg=0.858025 VRAM=38.8GiB | 23.5% done | ETA(epoch): 884s + [Epoch 117][30/66] loss=0.823202 avg=0.858649 VRAM=39.0GiB | 23.5% done | ETA(epoch): 691s + [Epoch 117][40/66] loss=0.837979 avg=0.855236 VRAM=38.8GiB | 23.5% done | ETA(epoch): 499s + [Epoch 117][50/66] loss=0.900258 avg=0.850651 VRAM=39.0GiB | 23.6% done | ETA(epoch): 307s + [Epoch 117][60/66] loss=0.861663 avg=0.850871 VRAM=38.8GiB | 23.6% done | ETA(epoch): 115s + Train loss: 0.850524 (1267.5s) ETA: 8823min + Val loss: 0.850277 [t_0.0-0.2=1.0584 t_0.2-0.4=0.9955 t_0.4-0.6=0.8514 t_0.6-0.8=0.7165 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 118/499 (24% done) --- + [Epoch 118][10/66] loss=0.907476 avg=0.868020 VRAM=39.0GiB | 23.6% done | ETA(epoch): 1074s + [Epoch 118][20/66] loss=0.913456 avg=0.867741 VRAM=38.8GiB | 23.7% done | ETA(epoch): 883s + [Epoch 118][30/66] loss=0.828329 avg=0.856401 VRAM=39.0GiB | 23.7% done | ETA(epoch): 691s + [Epoch 118][40/66] loss=0.949623 avg=0.850488 VRAM=38.8GiB | 23.7% done | ETA(epoch): 499s + [Epoch 118][50/66] loss=0.783666 avg=0.844596 VRAM=39.0GiB | 23.8% done | ETA(epoch): 307s + [Epoch 118][60/66] loss=0.803341 avg=0.845140 VRAM=38.8GiB | 23.8% done | ETA(epoch): 115s + Train loss: 0.851196 (1267.3s) ETA: 8799min + Val loss: 0.852101 [t_0.0-0.2=1.0629 t_0.2-0.4=1.0159 t_0.4-0.6=0.8484 t_0.6-0.8=0.7138 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 119/499 (24% done) --- + [Epoch 119][10/66] loss=0.886930 avg=0.886216 VRAM=39.0GiB | 23.8% done | ETA(epoch): 1075s + [Epoch 119][20/66] loss=0.888780 avg=0.894338 VRAM=38.8GiB | 23.9% done | ETA(epoch): 893s + [Epoch 119][30/66] loss=0.864113 avg=0.873565 VRAM=39.0GiB | 23.9% done | ETA(epoch): 697s + [Epoch 119][40/66] loss=0.887808 avg=0.870071 VRAM=38.8GiB | 23.9% done | ETA(epoch): 502s + [Epoch 119][50/66] loss=0.679286 avg=0.864441 VRAM=39.0GiB | 24.0% done | ETA(epoch): 309s + [Epoch 119][60/66] loss=0.799242 avg=0.865203 VRAM=38.8GiB | 24.0% done | ETA(epoch): 116s + Train loss: 0.863524 (1272.4s) ETA: 8775min + Val loss: 0.834902 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9900 t_0.4-0.6=0.8276 t_0.6-0.8=0.7213 t_0.8-1.0=0.6682] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 120/499 (24% done) --- + [Epoch 120][10/66] loss=0.977476 avg=0.848127 VRAM=39.0GiB | 24.0% done | ETA(epoch): 1089s + [Epoch 120][20/66] loss=0.901102 avg=0.859039 VRAM=38.8GiB | 24.1% done | ETA(epoch): 889s + [Epoch 120][30/66] loss=0.819311 avg=0.849220 VRAM=39.0GiB | 24.1% done | ETA(epoch): 695s + [Epoch 120][40/66] loss=0.805153 avg=0.844632 VRAM=38.8GiB | 24.1% done | ETA(epoch): 501s + [Epoch 120][50/66] loss=0.878296 avg=0.850844 VRAM=39.0GiB | 24.2% done | ETA(epoch): 308s + [Epoch 120][60/66] loss=0.870254 avg=0.847848 VRAM=38.8GiB | 24.2% done | ETA(epoch): 116s + Train loss: 0.852736 (1273.9s) ETA: 8751min + Val loss: 0.850503 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9997 t_0.4-0.6=0.8568 t_0.6-0.8=0.6963 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 17.7/188.5 GiB (9.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 121/499 (24% done) --- + [Epoch 121][10/66] loss=0.818806 avg=0.852067 VRAM=39.0GiB | 24.2% done | ETA(epoch): 1075s + [Epoch 121][20/66] loss=0.824306 avg=0.846283 VRAM=38.8GiB | 24.3% done | ETA(epoch): 883s + [Epoch 121][30/66] loss=0.685218 avg=0.837855 VRAM=39.0GiB | 24.3% done | ETA(epoch): 691s + [Epoch 121][40/66] loss=0.799370 avg=0.827266 VRAM=38.8GiB | 24.3% done | ETA(epoch): 499s + [Epoch 121][50/66] loss=0.858504 avg=0.833916 VRAM=39.0GiB | 24.4% done | ETA(epoch): 307s + [Epoch 121][60/66] loss=0.719538 avg=0.833993 VRAM=38.8GiB | 24.4% done | ETA(epoch): 115s + Train loss: 0.833664 (1269.6s) ETA: 8727min + Val loss: 0.828006 [t_0.0-0.2=1.0563 t_0.2-0.4=1.0102 t_0.4-0.6=0.8510 t_0.6-0.8=0.6865 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 122/499 (24% done) --- + [Epoch 122][10/66] loss=0.829776 avg=0.866280 VRAM=39.0GiB | 24.4% done | ETA(epoch): 1074s + [Epoch 122][20/66] loss=0.876875 avg=0.875978 VRAM=38.8GiB | 24.5% done | ETA(epoch): 884s + [Epoch 122][30/66] loss=0.955004 avg=0.871490 VRAM=39.0GiB | 24.5% done | ETA(epoch): 692s + [Epoch 122][40/66] loss=0.771851 avg=0.865313 VRAM=38.8GiB | 24.5% done | ETA(epoch): 499s + [Epoch 122][50/66] loss=0.904528 avg=0.865567 VRAM=39.0GiB | 24.6% done | ETA(epoch): 308s + [Epoch 122][60/66] loss=0.896451 avg=0.860871 VRAM=38.8GiB | 24.6% done | ETA(epoch): 115s + Train loss: 0.863959 (1270.1s) ETA: 8703min + Val loss: 0.850251 [t_0.0-0.2=1.0852 t_0.2-0.4=1.0003 t_0.4-0.6=0.8266 t_0.6-0.8=0.6966 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 123/499 (25% done) --- + [Epoch 123][10/66] loss=0.971855 avg=0.883742 VRAM=39.0GiB | 24.6% done | ETA(epoch): 1075s + [Epoch 123][20/66] loss=0.792109 avg=0.858514 VRAM=38.8GiB | 24.7% done | ETA(epoch): 883s + [Epoch 123][30/66] loss=1.028407 avg=0.856667 VRAM=39.0GiB | 24.7% done | ETA(epoch): 691s + [Epoch 123][40/66] loss=0.696180 avg=0.850969 VRAM=38.8GiB | 24.7% done | ETA(epoch): 499s + [Epoch 123][50/66] loss=0.957085 avg=0.859899 VRAM=39.0GiB | 24.8% done | ETA(epoch): 307s + [Epoch 123][60/66] loss=0.846704 avg=0.862057 VRAM=38.8GiB | 24.8% done | ETA(epoch): 115s + Train loss: 0.860848 (1267.3s) ETA: 8678min + Val loss: 0.889338 [t_0.0-0.2=1.0759 t_0.2-0.4=1.0023 t_0.4-0.6=0.8392 t_0.6-0.8=0.7097 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 124/499 (25% done) --- + [Epoch 124][10/66] loss=0.860966 avg=0.851828 VRAM=39.0GiB | 24.8% done | ETA(epoch): 1075s + [Epoch 124][20/66] loss=0.882132 avg=0.851620 VRAM=38.8GiB | 24.9% done | ETA(epoch): 883s + [Epoch 124][30/66] loss=0.791450 avg=0.845738 VRAM=39.0GiB | 24.9% done | ETA(epoch): 691s + [Epoch 124][40/66] loss=0.792549 avg=0.852484 VRAM=38.8GiB | 24.9% done | ETA(epoch): 499s + [Epoch 124][50/66] loss=0.958516 avg=0.856283 VRAM=39.0GiB | 25.0% done | ETA(epoch): 307s + [Epoch 124][60/66] loss=0.803426 avg=0.857082 VRAM=38.8GiB | 25.0% done | ETA(epoch): 115s + Train loss: 0.857316 (1267.9s) ETA: 8654min + Val loss: 0.858119 [t_0.0-0.2=1.0635 t_0.2-0.4=0.9982 t_0.4-0.6=0.8447 t_0.6-0.8=0.6906 t_0.8-1.0=0.6411] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 125/499 (25% done) --- + [Epoch 125][10/66] loss=0.989715 avg=0.828775 VRAM=39.0GiB | 25.0% done | ETA(epoch): 1075s + [Epoch 125][20/66] loss=0.815688 avg=0.843320 VRAM=38.8GiB | 25.1% done | ETA(epoch): 883s + [Epoch 125][30/66] loss=0.924196 avg=0.858077 VRAM=39.0GiB | 25.1% done | ETA(epoch): 691s + [Epoch 125][40/66] loss=0.829344 avg=0.858376 VRAM=38.8GiB | 25.1% done | ETA(epoch): 499s + [Epoch 125][50/66] loss=0.954778 avg=0.854247 VRAM=39.0GiB | 25.2% done | ETA(epoch): 307s + [Epoch 125][60/66] loss=0.770117 avg=0.848855 VRAM=38.8GiB | 25.2% done | ETA(epoch): 115s + Train loss: 0.853355 (1267.4s) ETA: 8630min + Val loss: 0.880388 [t_0.0-0.2=1.0770 t_0.2-0.4=1.0169 t_0.4-0.6=0.8468 t_0.6-0.8=0.7131 t_0.8-1.0=0.6410] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 126/499 (25% done) --- + [Epoch 126][10/66] loss=0.641196 avg=0.849307 VRAM=39.0GiB | 25.2% done | ETA(epoch): 1075s + [Epoch 126][20/66] loss=0.757632 avg=0.838234 VRAM=38.8GiB | 25.3% done | ETA(epoch): 888s + [Epoch 126][30/66] loss=0.834674 avg=0.847710 VRAM=39.0GiB | 25.3% done | ETA(epoch): 693s + [Epoch 126][40/66] loss=0.993577 avg=0.854352 VRAM=38.8GiB | 25.3% done | ETA(epoch): 501s + [Epoch 126][50/66] loss=0.867233 avg=0.852613 VRAM=39.0GiB | 25.4% done | ETA(epoch): 308s + [Epoch 126][60/66] loss=0.800418 avg=0.846539 VRAM=38.8GiB | 25.4% done | ETA(epoch): 115s + Train loss: 0.844730 (1269.6s) ETA: 8606min + Val loss: 0.856758 [t_0.0-0.2=1.0693 t_0.2-0.4=1.0015 t_0.4-0.6=0.8614 t_0.6-0.8=0.6992 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 127/499 (25% done) --- + [Epoch 127][10/66] loss=0.903555 avg=0.822776 VRAM=39.0GiB | 25.4% done | ETA(epoch): 1076s + [Epoch 127][20/66] loss=0.783558 avg=0.824853 VRAM=38.8GiB | 25.5% done | ETA(epoch): 884s + [Epoch 127][30/66] loss=0.782189 avg=0.829657 VRAM=39.0GiB | 25.5% done | ETA(epoch): 692s + [Epoch 127][40/66] loss=0.955812 avg=0.824494 VRAM=38.8GiB | 25.5% done | ETA(epoch): 500s + [Epoch 127][50/66] loss=0.823321 avg=0.828605 VRAM=39.0GiB | 25.6% done | ETA(epoch): 308s + [Epoch 127][60/66] loss=0.851488 avg=0.834627 VRAM=38.8GiB | 25.6% done | ETA(epoch): 115s + Train loss: 0.835465 (1268.8s) ETA: 8582min + Val loss: 0.866796 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9955 t_0.4-0.6=0.8335 t_0.6-0.8=0.7238 t_0.8-1.0=0.6237] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 128/499 (26% done) --- + [Epoch 128][10/66] loss=0.745855 avg=0.861855 VRAM=39.0GiB | 25.6% done | ETA(epoch): 1076s + [Epoch 128][20/66] loss=0.910407 avg=0.875959 VRAM=38.8GiB | 25.7% done | ETA(epoch): 884s + [Epoch 128][30/66] loss=0.862325 avg=0.884640 VRAM=39.0GiB | 25.7% done | ETA(epoch): 692s + [Epoch 128][40/66] loss=0.816848 avg=0.881381 VRAM=38.8GiB | 25.7% done | ETA(epoch): 500s + [Epoch 128][50/66] loss=0.902512 avg=0.874551 VRAM=39.0GiB | 25.8% done | ETA(epoch): 308s + [Epoch 128][60/66] loss=0.836618 avg=0.866309 VRAM=38.8GiB | 25.8% done | ETA(epoch): 115s + Train loss: 0.866688 (1268.4s) ETA: 8558min + Val loss: 0.863268 [t_0.0-0.2=1.0757 t_0.2-0.4=1.0156 t_0.4-0.6=0.8686 t_0.6-0.8=0.6919 t_0.8-1.0=0.6601] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 18.8/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 129/499 (26% done) --- + [Epoch 129][10/66] loss=0.778270 avg=0.823751 VRAM=39.0GiB | 25.8% done | ETA(epoch): 1076s + [Epoch 129][20/66] loss=0.785893 avg=0.832733 VRAM=38.8GiB | 25.9% done | ETA(epoch): 885s + [Epoch 129][30/66] loss=0.858681 avg=0.848009 VRAM=39.0GiB | 25.9% done | ETA(epoch): 692s + [Epoch 129][40/66] loss=0.847059 avg=0.839563 VRAM=38.8GiB | 25.9% done | ETA(epoch): 500s + [Epoch 129][50/66] loss=0.778876 avg=0.842552 VRAM=39.0GiB | 26.0% done | ETA(epoch): 308s + [Epoch 129][60/66] loss=0.802168 avg=0.839947 VRAM=38.8GiB | 26.0% done | ETA(epoch): 115s + Train loss: 0.839013 (1268.5s) ETA: 8534min + Val loss: 0.825512 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0136 t_0.4-0.6=0.8127 t_0.6-0.8=0.6988 t_0.8-1.0=0.6494] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 130/499 (26% done) --- + [Epoch 130][10/66] loss=0.886927 avg=0.828859 VRAM=39.0GiB | 26.0% done | ETA(epoch): 1075s + [Epoch 130][20/66] loss=0.945453 avg=0.841794 VRAM=38.8GiB | 26.1% done | ETA(epoch): 883s + [Epoch 130][30/66] loss=0.926144 avg=0.862630 VRAM=39.0GiB | 26.1% done | ETA(epoch): 691s + [Epoch 130][40/66] loss=0.898956 avg=0.865936 VRAM=38.8GiB | 26.1% done | ETA(epoch): 499s + [Epoch 130][50/66] loss=0.892774 avg=0.868494 VRAM=39.0GiB | 26.2% done | ETA(epoch): 307s + [Epoch 130][60/66] loss=0.993802 avg=0.869376 VRAM=38.8GiB | 26.2% done | ETA(epoch): 115s + Train loss: 0.867696 (1267.4s) ETA: 8510min + Val loss: 0.833677 [t_0.0-0.2=1.0527 t_0.2-0.4=1.0201 t_0.4-0.6=0.8445 t_0.6-0.8=0.7326 t_0.8-1.0=0.6335] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 131/499 (26% done) --- + [Epoch 131][10/66] loss=0.857881 avg=0.843442 VRAM=39.0GiB | 26.2% done | ETA(epoch): 1075s + [Epoch 131][20/66] loss=0.841923 avg=0.829465 VRAM=38.8GiB | 26.3% done | ETA(epoch): 884s + [Epoch 131][30/66] loss=0.848973 avg=0.854508 VRAM=39.0GiB | 26.3% done | ETA(epoch): 692s + [Epoch 131][40/66] loss=0.778863 avg=0.850236 VRAM=38.8GiB | 26.3% done | ETA(epoch): 501s + [Epoch 131][50/66] loss=0.916162 avg=0.855187 VRAM=39.0GiB | 26.4% done | ETA(epoch): 308s + [Epoch 131][60/66] loss=0.859443 avg=0.856263 VRAM=38.8GiB | 26.4% done | ETA(epoch): 115s + Train loss: 0.855682 (1270.5s) ETA: 8486min + Val loss: 0.880526 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9952 t_0.4-0.6=0.8234 t_0.6-0.8=0.7129 t_0.8-1.0=0.6490] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 132/499 (26% done) --- + [Epoch 132][10/66] loss=0.760340 avg=0.834923 VRAM=39.0GiB | 26.4% done | ETA(epoch): 1075s + [Epoch 132][20/66] loss=0.983509 avg=0.835494 VRAM=38.8GiB | 26.5% done | ETA(epoch): 884s + [Epoch 132][30/66] loss=0.840346 avg=0.845557 VRAM=39.0GiB | 26.5% done | ETA(epoch): 692s + [Epoch 132][40/66] loss=0.919896 avg=0.845169 VRAM=38.8GiB | 26.5% done | ETA(epoch): 499s + [Epoch 132][50/66] loss=0.731207 avg=0.845237 VRAM=39.0GiB | 26.6% done | ETA(epoch): 307s + [Epoch 132][60/66] loss=0.983301 avg=0.858591 VRAM=38.8GiB | 26.6% done | ETA(epoch): 115s + Train loss: 0.858851 (1268.0s) ETA: 8462min + Val loss: 0.836217 [t_0.0-0.2=1.0608 t_0.2-0.4=0.9740 t_0.4-0.6=0.8587 t_0.6-0.8=0.6988 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 133/499 (27% done) --- + [Epoch 133][10/66] loss=0.713354 avg=0.855367 VRAM=39.0GiB | 26.6% done | ETA(epoch): 1076s + [Epoch 133][20/66] loss=0.796997 avg=0.861743 VRAM=38.8GiB | 26.7% done | ETA(epoch): 884s + [Epoch 133][30/66] loss=0.730050 avg=0.851363 VRAM=39.0GiB | 26.7% done | ETA(epoch): 692s + [Epoch 133][40/66] loss=0.853266 avg=0.850425 VRAM=38.8GiB | 26.7% done | ETA(epoch): 500s + [Epoch 133][50/66] loss=1.010185 avg=0.850716 VRAM=39.0GiB | 26.8% done | ETA(epoch): 307s + [Epoch 133][60/66] loss=0.856374 avg=0.853620 VRAM=38.8GiB | 26.8% done | ETA(epoch): 115s + Train loss: 0.853817 (1268.5s) ETA: 8438min + Val loss: 0.846846 [t_0.0-0.2=1.0602 t_0.2-0.4=1.0220 t_0.4-0.6=0.8572 t_0.6-0.8=0.7175 t_0.8-1.0=0.6452] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 18.9/188.5 GiB (10.0%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 134/499 (27% done) --- + [Epoch 134][10/66] loss=0.878929 avg=0.795070 VRAM=39.0GiB | 26.8% done | ETA(epoch): 1075s + [Epoch 134][20/66] loss=0.863167 avg=0.812899 VRAM=38.8GiB | 26.9% done | ETA(epoch): 883s + [Epoch 134][30/66] loss=0.767857 avg=0.811901 VRAM=39.0GiB | 26.9% done | ETA(epoch): 691s + [Epoch 134][40/66] loss=0.879586 avg=0.812337 VRAM=38.8GiB | 26.9% done | ETA(epoch): 499s + [Epoch 134][50/66] loss=0.820307 avg=0.831910 VRAM=39.0GiB | 27.0% done | ETA(epoch): 307s + [Epoch 134][60/66] loss=0.836512 avg=0.841806 VRAM=38.8GiB | 27.0% done | ETA(epoch): 115s + Train loss: 0.844141 (1267.2s) ETA: 8414min + Val loss: 0.840763 [t_0.0-0.2=1.0860 t_0.2-0.4=0.9835 t_0.4-0.6=0.8429 t_0.6-0.8=0.6925 t_0.8-1.0=0.6331] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 135/499 (27% done) --- + [Epoch 135][10/66] loss=0.892047 avg=0.847496 VRAM=39.0GiB | 27.0% done | ETA(epoch): 1097s + [Epoch 135][20/66] loss=0.710606 avg=0.840071 VRAM=38.8GiB | 27.1% done | ETA(epoch): 892s + [Epoch 135][30/66] loss=0.734758 avg=0.841335 VRAM=39.0GiB | 27.1% done | ETA(epoch): 696s + [Epoch 135][40/66] loss=0.750583 avg=0.845293 VRAM=38.8GiB | 27.1% done | ETA(epoch): 501s + [Epoch 135][50/66] loss=0.809885 avg=0.844021 VRAM=39.0GiB | 27.2% done | ETA(epoch): 308s + [Epoch 135][60/66] loss=0.894676 avg=0.850093 VRAM=38.8GiB | 27.2% done | ETA(epoch): 116s + Train loss: 0.851124 (1271.2s) ETA: 8390min + Val loss: 0.860508 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9792 t_0.4-0.6=0.8199 t_0.6-0.8=0.7130 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 17.4/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 136/499 (27% done) --- + [Epoch 136][10/66] loss=0.949226 avg=0.847706 VRAM=39.0GiB | 27.2% done | ETA(epoch): 1075s + [Epoch 136][20/66] loss=0.963510 avg=0.860464 VRAM=38.8GiB | 27.3% done | ETA(epoch): 883s + [Epoch 136][30/66] loss=0.870965 avg=0.886287 VRAM=39.0GiB | 27.3% done | ETA(epoch): 691s + [Epoch 136][40/66] loss=0.828817 avg=0.872909 VRAM=38.8GiB | 27.3% done | ETA(epoch): 499s + [Epoch 136][50/66] loss=0.751328 avg=0.870720 VRAM=39.0GiB | 27.4% done | ETA(epoch): 307s + [Epoch 136][60/66] loss=0.937213 avg=0.863477 VRAM=38.8GiB | 27.4% done | ETA(epoch): 115s + Train loss: 0.864089 (1267.7s) ETA: 8366min + Val loss: 0.848725 [t_0.0-0.2=1.0667 t_0.2-0.4=1.0014 t_0.4-0.6=0.8529 t_0.6-0.8=0.6995 t_0.8-1.0=0.6528] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 137/499 (27% done) --- + [Epoch 137][10/66] loss=0.893269 avg=0.869888 VRAM=39.0GiB | 27.4% done | ETA(epoch): 1076s + [Epoch 137][20/66] loss=0.869905 avg=0.854605 VRAM=38.8GiB | 27.5% done | ETA(epoch): 884s + [Epoch 137][30/66] loss=0.777482 avg=0.863240 VRAM=39.0GiB | 27.5% done | ETA(epoch): 692s + [Epoch 137][40/66] loss=0.818964 avg=0.857176 VRAM=38.8GiB | 27.5% done | ETA(epoch): 500s + [Epoch 137][50/66] loss=0.940608 avg=0.859071 VRAM=39.0GiB | 27.6% done | ETA(epoch): 307s + [Epoch 137][60/66] loss=0.923853 avg=0.851418 VRAM=38.8GiB | 27.6% done | ETA(epoch): 115s + Train loss: 0.854974 (1267.9s) ETA: 8342min + Val loss: 0.869103 [t_0.0-0.2=1.0690 t_0.2-0.4=0.9673 t_0.4-0.6=0.8856 t_0.6-0.8=0.7143 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 17.6/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 138/499 (28% done) --- + [Epoch 138][10/66] loss=0.919253 avg=0.851178 VRAM=39.0GiB | 27.6% done | ETA(epoch): 1075s + [Epoch 138][20/66] loss=0.900421 avg=0.836277 VRAM=38.8GiB | 27.7% done | ETA(epoch): 884s + [Epoch 138][30/66] loss=0.718219 avg=0.836458 VRAM=39.0GiB | 27.7% done | ETA(epoch): 692s + [Epoch 138][40/66] loss=0.911601 avg=0.845629 VRAM=38.8GiB | 27.7% done | ETA(epoch): 499s + [Epoch 138][50/66] loss=0.856315 avg=0.851081 VRAM=39.0GiB | 27.8% done | ETA(epoch): 308s + [Epoch 138][60/66] loss=0.736735 avg=0.852161 VRAM=38.8GiB | 27.8% done | ETA(epoch): 116s + Train loss: 0.850018 (1270.3s) ETA: 8319min + Val loss: 0.809739 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9883 t_0.4-0.6=0.8451 t_0.6-0.8=0.7024 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 139/499 (28% done) --- + [Epoch 139][10/66] loss=0.790470 avg=0.854419 VRAM=39.0GiB | 27.8% done | ETA(epoch): 1074s + [Epoch 139][20/66] loss=0.972273 avg=0.859797 VRAM=38.8GiB | 27.9% done | ETA(epoch): 883s + [Epoch 139][30/66] loss=0.903834 avg=0.871741 VRAM=39.0GiB | 27.9% done | ETA(epoch): 691s + [Epoch 139][40/66] loss=0.820424 avg=0.862904 VRAM=38.8GiB | 27.9% done | ETA(epoch): 502s + [Epoch 139][50/66] loss=1.023295 avg=0.863149 VRAM=39.0GiB | 28.0% done | ETA(epoch): 308s + [Epoch 139][60/66] loss=0.792039 avg=0.862115 VRAM=38.8GiB | 28.0% done | ETA(epoch): 116s + Train loss: 0.858517 (1271.0s) ETA: 8295min + Val loss: 0.834392 [t_0.0-0.2=1.0871 t_0.2-0.4=0.9936 t_0.4-0.6=0.8273 t_0.6-0.8=0.7136 t_0.8-1.0=0.6329] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 140/499 (28% done) --- + [Epoch 140][10/66] loss=0.948664 avg=0.888686 VRAM=39.0GiB | 28.0% done | ETA(epoch): 1074s + [Epoch 140][20/66] loss=0.867716 avg=0.854183 VRAM=38.8GiB | 28.1% done | ETA(epoch): 883s + [Epoch 140][30/66] loss=0.820722 avg=0.864838 VRAM=39.0GiB | 28.1% done | ETA(epoch): 691s + [Epoch 140][40/66] loss=0.804824 avg=0.868522 VRAM=38.8GiB | 28.1% done | ETA(epoch): 499s + [Epoch 140][50/66] loss=0.730382 avg=0.877630 VRAM=39.0GiB | 28.2% done | ETA(epoch): 307s + [Epoch 140][60/66] loss=0.816502 avg=0.866802 VRAM=38.8GiB | 28.2% done | ETA(epoch): 115s + Train loss: 0.861200 (1267.9s) ETA: 8271min + Val loss: 0.894965 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9989 t_0.4-0.6=0.8510 t_0.6-0.8=0.6931 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 141/499 (28% done) --- + [Epoch 141][10/66] loss=0.876951 avg=0.828740 VRAM=39.0GiB | 28.2% done | ETA(epoch): 1075s + [Epoch 141][20/66] loss=0.918387 avg=0.849549 VRAM=38.8GiB | 28.3% done | ETA(epoch): 883s + [Epoch 141][30/66] loss=0.912527 avg=0.858976 VRAM=39.0GiB | 28.3% done | ETA(epoch): 691s + [Epoch 141][40/66] loss=0.738081 avg=0.855215 VRAM=38.8GiB | 28.3% done | ETA(epoch): 499s + [Epoch 141][50/66] loss=0.908763 avg=0.854974 VRAM=39.0GiB | 28.4% done | ETA(epoch): 308s + [Epoch 141][60/66] loss=0.880725 avg=0.852032 VRAM=38.8GiB | 28.4% done | ETA(epoch): 115s + Train loss: 0.853958 (1268.6s) ETA: 8247min + Val loss: 0.849171 [t_0.0-0.2=1.0875 t_0.2-0.4=1.0097 t_0.4-0.6=0.8579 t_0.6-0.8=0.7113 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 17.4/188.5 GiB (9.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 142/499 (28% done) --- + [Epoch 142][10/66] loss=0.940206 avg=0.815703 VRAM=39.0GiB | 28.4% done | ETA(epoch): 1074s + [Epoch 142][20/66] loss=0.939119 avg=0.833328 VRAM=38.8GiB | 28.5% done | ETA(epoch): 882s + [Epoch 142][30/66] loss=0.794648 avg=0.823033 VRAM=39.0GiB | 28.5% done | ETA(epoch): 691s + [Epoch 142][40/66] loss=0.914537 avg=0.836388 VRAM=38.8GiB | 28.5% done | ETA(epoch): 499s + [Epoch 142][50/66] loss=0.857784 avg=0.832614 VRAM=39.0GiB | 28.6% done | ETA(epoch): 307s + [Epoch 142][60/66] loss=0.842475 avg=0.839229 VRAM=38.8GiB | 28.6% done | ETA(epoch): 115s + Train loss: 0.844826 (1267.0s) ETA: 8223min + Val loss: 0.853349 [t_0.0-0.2=1.0882 t_0.2-0.4=0.9836 t_0.4-0.6=0.8445 t_0.6-0.8=0.7539 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 17.5/188.5 GiB (9.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 143/499 (29% done) --- + [Epoch 143][10/66] loss=1.007013 avg=0.853111 VRAM=39.0GiB | 28.6% done | ETA(epoch): 1092s + [Epoch 143][20/66] loss=0.810502 avg=0.860705 VRAM=38.8GiB | 28.7% done | ETA(epoch): 891s + [Epoch 143][30/66] loss=0.716324 avg=0.863473 VRAM=39.0GiB | 28.7% done | ETA(epoch): 695s + [Epoch 143][40/66] loss=0.975418 avg=0.855967 VRAM=38.8GiB | 28.7% done | ETA(epoch): 501s + [Epoch 143][50/66] loss=0.875871 avg=0.856425 VRAM=39.0GiB | 28.8% done | ETA(epoch): 308s + [Epoch 143][60/66] loss=0.815729 avg=0.852830 VRAM=38.8GiB | 28.8% done | ETA(epoch): 116s + Train loss: 0.852003 (1272.7s) ETA: 8200min + Val loss: 0.866363 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0092 t_0.4-0.6=0.8520 t_0.6-0.8=0.6959 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 17.8/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 144/499 (29% done) --- + [Epoch 144][10/66] loss=0.784218 avg=0.824604 VRAM=39.0GiB | 28.8% done | ETA(epoch): 1074s + [Epoch 144][20/66] loss=0.797053 avg=0.844306 VRAM=38.8GiB | 28.9% done | ETA(epoch): 883s + [Epoch 144][30/66] loss=0.885237 avg=0.848607 VRAM=39.0GiB | 28.9% done | ETA(epoch): 691s + [Epoch 144][40/66] loss=0.728571 avg=0.840699 VRAM=38.8GiB | 28.9% done | ETA(epoch): 499s + [Epoch 144][50/66] loss=0.896378 avg=0.835341 VRAM=39.0GiB | 29.0% done | ETA(epoch): 307s + [Epoch 144][60/66] loss=0.785885 avg=0.835679 VRAM=38.8GiB | 29.0% done | ETA(epoch): 115s + Train loss: 0.839850 (1269.2s) ETA: 8176min + Val loss: 0.846144 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9800 t_0.4-0.6=0.8615 t_0.6-0.8=0.7022 t_0.8-1.0=0.6312] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 145/499 (29% done) --- + [Epoch 145][10/66] loss=0.805421 avg=0.872459 VRAM=39.0GiB | 29.0% done | ETA(epoch): 1075s + [Epoch 145][20/66] loss=0.716269 avg=0.859373 VRAM=38.8GiB | 29.1% done | ETA(epoch): 884s + [Epoch 145][30/66] loss=0.965377 avg=0.874187 VRAM=39.0GiB | 29.1% done | ETA(epoch): 692s + [Epoch 145][40/66] loss=0.920182 avg=0.864874 VRAM=38.8GiB | 29.1% done | ETA(epoch): 500s + [Epoch 145][50/66] loss=0.809632 avg=0.852488 VRAM=39.0GiB | 29.2% done | ETA(epoch): 307s + [Epoch 145][60/66] loss=0.881783 avg=0.847601 VRAM=38.8GiB | 29.2% done | ETA(epoch): 115s + Train loss: 0.848925 (1268.5s) ETA: 8152min + Val loss: 0.859137 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9959 t_0.4-0.6=0.8437 t_0.6-0.8=0.6979 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 18.0/188.5 GiB (9.6%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 146/499 (29% done) --- + [Epoch 146][10/66] loss=0.745014 avg=0.841443 VRAM=39.0GiB | 29.2% done | ETA(epoch): 1075s + [Epoch 146][20/66] loss=0.770802 avg=0.855891 VRAM=38.8GiB | 29.3% done | ETA(epoch): 884s + [Epoch 146][30/66] loss=0.841335 avg=0.848260 VRAM=39.0GiB | 29.3% done | ETA(epoch): 692s + [Epoch 146][40/66] loss=0.734724 avg=0.841497 VRAM=38.8GiB | 29.3% done | ETA(epoch): 499s + [Epoch 146][50/66] loss=0.804644 avg=0.842525 VRAM=39.0GiB | 29.4% done | ETA(epoch): 307s + [Epoch 146][60/66] loss=0.893195 avg=0.845173 VRAM=38.8GiB | 29.4% done | ETA(epoch): 115s + Train loss: 0.847440 (1268.1s) ETA: 8128min + Val loss: 0.862572 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9655 t_0.4-0.6=0.8388 t_0.6-0.8=0.7273 t_0.8-1.0=0.6364] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 18.0/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 147/499 (29% done) --- + [Epoch 147][10/66] loss=0.847722 avg=0.819850 VRAM=39.0GiB | 29.4% done | ETA(epoch): 1074s + [Epoch 147][20/66] loss=0.922290 avg=0.820887 VRAM=38.8GiB | 29.5% done | ETA(epoch): 883s + [Epoch 147][30/66] loss=0.817927 avg=0.829767 VRAM=39.0GiB | 29.5% done | ETA(epoch): 691s + [Epoch 147][40/66] loss=0.852517 avg=0.827056 VRAM=38.8GiB | 29.5% done | ETA(epoch): 499s + [Epoch 147][50/66] loss=0.846113 avg=0.833070 VRAM=39.0GiB | 29.6% done | ETA(epoch): 307s + [Epoch 147][60/66] loss=0.951509 avg=0.833952 VRAM=38.8GiB | 29.6% done | ETA(epoch): 115s + Train loss: 0.831335 (1267.1s) ETA: 8105min + Val loss: 0.838487 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9890 t_0.4-0.6=0.8336 t_0.6-0.8=0.7229 t_0.8-1.0=0.6333] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 148/499 (30% done) --- + [Epoch 148][10/66] loss=0.889264 avg=0.891379 VRAM=39.0GiB | 29.6% done | ETA(epoch): 1078s + [Epoch 148][20/66] loss=0.963634 avg=0.865250 VRAM=38.8GiB | 29.7% done | ETA(epoch): 884s + [Epoch 148][30/66] loss=0.865209 avg=0.850093 VRAM=39.0GiB | 29.7% done | ETA(epoch): 692s + [Epoch 148][40/66] loss=0.723419 avg=0.854920 VRAM=38.8GiB | 29.7% done | ETA(epoch): 500s + [Epoch 148][50/66] loss=0.928880 avg=0.857607 VRAM=39.0GiB | 29.8% done | ETA(epoch): 307s + [Epoch 148][60/66] loss=1.082087 avg=0.862792 VRAM=38.8GiB | 29.8% done | ETA(epoch): 115s + Train loss: 0.860209 (1268.0s) ETA: 8081min + Val loss: 0.831646 [t_0.0-0.2=1.0614 t_0.2-0.4=0.9841 t_0.4-0.6=0.8536 t_0.6-0.8=0.7085 t_0.8-1.0=0.6420] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 149/499 (30% done) --- + [Epoch 149][10/66] loss=0.785718 avg=0.891085 VRAM=39.0GiB | 29.8% done | ETA(epoch): 1074s + [Epoch 149][20/66] loss=0.942487 avg=0.878797 VRAM=38.8GiB | 29.9% done | ETA(epoch): 882s + [Epoch 149][30/66] loss=0.886860 avg=0.882600 VRAM=39.0GiB | 29.9% done | ETA(epoch): 691s + [Epoch 149][40/66] loss=0.872178 avg=0.876756 VRAM=38.8GiB | 29.9% done | ETA(epoch): 499s + [Epoch 149][50/66] loss=0.794636 avg=0.874230 VRAM=39.0GiB | 30.0% done | ETA(epoch): 307s + [Epoch 149][60/66] loss=0.901381 avg=0.877770 VRAM=38.8GiB | 30.0% done | ETA(epoch): 115s + Train loss: 0.878006 (1266.9s) ETA: 8057min + Val loss: 0.835006 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9740 t_0.4-0.6=0.8397 t_0.6-0.8=0.7207 t_0.8-1.0=0.6249] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 17.9/188.5 GiB (9.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 150/499 (30% done) --- + [MilestoneVis] train_0 step 9900 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9900 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9900 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9900 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9900 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9900 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9900 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9900 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9900 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9900 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/66] loss=0.957451 avg=0.884364 VRAM=39.0GiB | 30.0% done | ETA(epoch): 6846s + [Epoch 150][20/66] loss=0.864592 avg=0.865774 VRAM=38.8GiB | 30.1% done | ETA(epoch): 3266s + [Epoch 150][30/66] loss=0.828386 avg=0.847064 VRAM=39.0GiB | 30.1% done | ETA(epoch): 1935s + [Epoch 150][40/66] loss=0.741526 avg=0.844059 VRAM=38.8GiB | 30.1% done | ETA(epoch): 1173s + [Epoch 150][50/66] loss=0.788111 avg=0.842338 VRAM=39.0GiB | 30.2% done | ETA(epoch): 639s + [Epoch 150][60/66] loss=0.943114 avg=0.850794 VRAM=38.8GiB | 30.2% done | ETA(epoch): 219s + Train loss: 0.850188 (2304.2s) ETA: 8073min + Val loss: 0.854280 [t_0.0-0.2=1.0625 t_0.2-0.4=1.0255 t_0.4-0.6=0.8207 t_0.6-0.8=0.6957 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 151/499 (30% done) --- + [Epoch 151][10/66] loss=0.829058 avg=0.852292 VRAM=39.0GiB | 30.2% done | ETA(epoch): 1077s + [Epoch 151][20/66] loss=0.802483 avg=0.851718 VRAM=38.8GiB | 30.3% done | ETA(epoch): 884s + [Epoch 151][30/66] loss=0.882716 avg=0.852570 VRAM=39.0GiB | 30.3% done | ETA(epoch): 692s + [Epoch 151][40/66] loss=0.802789 avg=0.852229 VRAM=38.8GiB | 30.3% done | ETA(epoch): 500s + [Epoch 151][50/66] loss=0.994278 avg=0.854872 VRAM=39.0GiB | 30.4% done | ETA(epoch): 307s + [Epoch 151][60/66] loss=0.752639 avg=0.851417 VRAM=38.8GiB | 30.4% done | ETA(epoch): 115s + Train loss: 0.848774 (1267.8s) ETA: 8049min + Val loss: 0.878739 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9844 t_0.4-0.6=0.8414 t_0.6-0.8=0.7114 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 152/499 (30% done) --- + [Epoch 152][10/66] loss=0.837759 avg=0.852818 VRAM=39.0GiB | 30.4% done | ETA(epoch): 1075s + [Epoch 152][20/66] loss=0.945987 avg=0.847556 VRAM=38.8GiB | 30.5% done | ETA(epoch): 883s + [Epoch 152][30/66] loss=0.700045 avg=0.850425 VRAM=39.0GiB | 30.5% done | ETA(epoch): 691s + [Epoch 152][40/66] loss=0.691533 avg=0.843724 VRAM=38.8GiB | 30.5% done | ETA(epoch): 499s + [Epoch 152][50/66] loss=0.769646 avg=0.847019 VRAM=39.0GiB | 30.6% done | ETA(epoch): 307s + [Epoch 152][60/66] loss=0.800878 avg=0.844636 VRAM=38.8GiB | 30.6% done | ETA(epoch): 115s + Train loss: 0.847413 (1268.0s) ETA: 8025min + Val loss: 0.820953 [t_0.0-0.2=1.0708 t_0.2-0.4=0.9613 t_0.4-0.6=0.8603 t_0.6-0.8=0.7030 t_0.8-1.0=0.6356] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 153/499 (31% done) --- + [Epoch 153][10/66] loss=0.699375 avg=0.852575 VRAM=39.0GiB | 30.6% done | ETA(epoch): 1074s + [Epoch 153][20/66] loss=0.904599 avg=0.875298 VRAM=38.8GiB | 30.7% done | ETA(epoch): 883s + [Epoch 153][30/66] loss=0.833155 avg=0.871181 VRAM=39.0GiB | 30.7% done | ETA(epoch): 691s + [Epoch 153][40/66] loss=0.827774 avg=0.860855 VRAM=38.8GiB | 30.7% done | ETA(epoch): 499s + [Epoch 153][50/66] loss=0.888784 avg=0.862574 VRAM=39.0GiB | 30.8% done | ETA(epoch): 307s + [Epoch 153][60/66] loss=0.717773 avg=0.861697 VRAM=38.8GiB | 30.8% done | ETA(epoch): 115s + Train loss: 0.862635 (1270.1s) ETA: 8001min + Val loss: 0.844745 [t_0.0-0.2=1.0808 t_0.2-0.4=1.0028 t_0.4-0.6=0.8384 t_0.6-0.8=0.7110 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 154/499 (31% done) --- + [Epoch 154][10/66] loss=0.871471 avg=0.870985 VRAM=39.0GiB | 30.8% done | ETA(epoch): 1076s + [Epoch 154][20/66] loss=0.889417 avg=0.863568 VRAM=38.8GiB | 30.9% done | ETA(epoch): 885s + [Epoch 154][30/66] loss=0.868095 avg=0.856787 VRAM=39.0GiB | 30.9% done | ETA(epoch): 692s + [Epoch 154][40/66] loss=0.807110 avg=0.858074 VRAM=38.8GiB | 30.9% done | ETA(epoch): 500s + [Epoch 154][50/66] loss=0.910923 avg=0.856609 VRAM=39.0GiB | 31.0% done | ETA(epoch): 308s + [Epoch 154][60/66] loss=0.772875 avg=0.858887 VRAM=38.8GiB | 31.0% done | ETA(epoch): 115s + Train loss: 0.858413 (1269.7s) ETA: 7977min + Val loss: 0.837708 [t_0.0-0.2=1.0635 t_0.2-0.4=0.9777 t_0.4-0.6=0.8599 t_0.6-0.8=0.7053 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 155/499 (31% done) --- + [Epoch 155][10/66] loss=0.757082 avg=0.841473 VRAM=39.0GiB | 31.0% done | ETA(epoch): 1076s + [Epoch 155][20/66] loss=0.908144 avg=0.853324 VRAM=38.8GiB | 31.1% done | ETA(epoch): 883s + [Epoch 155][30/66] loss=0.833203 avg=0.853029 VRAM=39.0GiB | 31.1% done | ETA(epoch): 692s + [Epoch 155][40/66] loss=0.902108 avg=0.851868 VRAM=38.8GiB | 31.1% done | ETA(epoch): 500s + [Epoch 155][50/66] loss=0.776918 avg=0.845015 VRAM=39.0GiB | 31.2% done | ETA(epoch): 308s + [Epoch 155][60/66] loss=0.770991 avg=0.847050 VRAM=38.8GiB | 31.2% done | ETA(epoch): 115s + Train loss: 0.847192 (1268.6s) ETA: 7953min + Val loss: 0.816412 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9734 t_0.4-0.6=0.8655 t_0.6-0.8=0.7174 t_0.8-1.0=0.6322] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 156/499 (31% done) --- + [Epoch 156][10/66] loss=0.929902 avg=0.893812 VRAM=39.0GiB | 31.2% done | ETA(epoch): 1075s + [Epoch 156][20/66] loss=0.850166 avg=0.855663 VRAM=38.8GiB | 31.3% done | ETA(epoch): 883s + [Epoch 156][30/66] loss=0.813871 avg=0.854942 VRAM=39.0GiB | 31.3% done | ETA(epoch): 692s + [Epoch 156][40/66] loss=0.940041 avg=0.851421 VRAM=38.8GiB | 31.3% done | ETA(epoch): 499s + [Epoch 156][50/66] loss=0.890100 avg=0.857058 VRAM=39.0GiB | 31.4% done | ETA(epoch): 307s + [Epoch 156][60/66] loss=0.948231 avg=0.853677 VRAM=38.8GiB | 31.4% done | ETA(epoch): 115s + Train loss: 0.855790 (1267.8s) ETA: 7929min + Val loss: 0.884890 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9976 t_0.4-0.6=0.8248 t_0.6-0.8=0.7110 t_0.8-1.0=0.6249] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 157/499 (31% done) --- + [Epoch 157][10/66] loss=0.814962 avg=0.866890 VRAM=39.0GiB | 31.4% done | ETA(epoch): 1078s + [Epoch 157][20/66] loss=0.822736 avg=0.859460 VRAM=38.8GiB | 31.5% done | ETA(epoch): 893s + [Epoch 157][30/66] loss=0.836464 avg=0.856221 VRAM=39.0GiB | 31.5% done | ETA(epoch): 697s + [Epoch 157][40/66] loss=0.868958 avg=0.849359 VRAM=38.8GiB | 31.5% done | ETA(epoch): 502s + [Epoch 157][50/66] loss=0.909713 avg=0.839467 VRAM=39.0GiB | 31.6% done | ETA(epoch): 309s + [Epoch 157][60/66] loss=0.816664 avg=0.843084 VRAM=38.8GiB | 31.6% done | ETA(epoch): 116s + Train loss: 0.843143 (1272.3s) ETA: 7905min + Val loss: 0.877149 [t_0.0-0.2=1.0630 t_0.2-0.4=1.0233 t_0.4-0.6=0.8492 t_0.6-0.8=0.6894 t_0.8-1.0=0.6472] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 158/499 (32% done) --- + [Epoch 158][10/66] loss=0.886670 avg=0.871366 VRAM=39.0GiB | 31.6% done | ETA(epoch): 1075s + [Epoch 158][20/66] loss=0.781091 avg=0.851727 VRAM=38.8GiB | 31.7% done | ETA(epoch): 888s + [Epoch 158][30/66] loss=0.855128 avg=0.862240 VRAM=39.0GiB | 31.7% done | ETA(epoch): 694s + [Epoch 158][40/66] loss=0.772693 avg=0.857938 VRAM=38.8GiB | 31.7% done | ETA(epoch): 501s + [Epoch 158][50/66] loss=0.845239 avg=0.852335 VRAM=39.0GiB | 31.8% done | ETA(epoch): 308s + [Epoch 158][60/66] loss=0.848698 avg=0.847856 VRAM=38.8GiB | 31.8% done | ETA(epoch): 115s + Train loss: 0.850330 (1269.4s) ETA: 7881min + Val loss: 0.825440 [t_0.0-0.2=1.0603 t_0.2-0.4=0.9963 t_0.4-0.6=0.8326 t_0.6-0.8=0.7090 t_0.8-1.0=0.6175] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 159/499 (32% done) --- + [Epoch 159][10/66] loss=0.875712 avg=0.802747 VRAM=39.0GiB | 31.8% done | ETA(epoch): 1085s + [Epoch 159][20/66] loss=0.796735 avg=0.843299 VRAM=38.8GiB | 31.9% done | ETA(epoch): 886s + [Epoch 159][30/66] loss=1.019482 avg=0.869907 VRAM=39.0GiB | 31.9% done | ETA(epoch): 693s + [Epoch 159][40/66] loss=0.851209 avg=0.865021 VRAM=38.8GiB | 31.9% done | ETA(epoch): 500s + [Epoch 159][50/66] loss=1.019109 avg=0.867743 VRAM=39.0GiB | 32.0% done | ETA(epoch): 308s + [Epoch 159][60/66] loss=0.804183 avg=0.861258 VRAM=38.8GiB | 32.0% done | ETA(epoch): 115s + Train loss: 0.861212 (1268.4s) ETA: 7857min + Val loss: 0.858728 [t_0.0-0.2=1.0688 t_0.2-0.4=1.0028 t_0.4-0.6=0.8021 t_0.6-0.8=0.7100 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 160/499 (32% done) --- + [Epoch 160][10/66] loss=0.820757 avg=0.860027 VRAM=39.0GiB | 32.0% done | ETA(epoch): 1074s + [Epoch 160][20/66] loss=0.903262 avg=0.851317 VRAM=38.8GiB | 32.1% done | ETA(epoch): 883s + [Epoch 160][30/66] loss=0.862035 avg=0.853727 VRAM=39.0GiB | 32.1% done | ETA(epoch): 692s + [Epoch 160][40/66] loss=0.819784 avg=0.849271 VRAM=38.8GiB | 32.1% done | ETA(epoch): 499s + [Epoch 160][50/66] loss=0.841867 avg=0.847513 VRAM=39.0GiB | 32.2% done | ETA(epoch): 307s + [Epoch 160][60/66] loss=0.913840 avg=0.843245 VRAM=38.8GiB | 32.2% done | ETA(epoch): 115s + Train loss: 0.845587 (1268.5s) ETA: 7834min + Val loss: 0.843698 [t_0.0-0.2=1.0660 t_0.2-0.4=1.0292 t_0.4-0.6=0.8111 t_0.6-0.8=0.7093 t_0.8-1.0=0.6235] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 161/499 (32% done) --- + [Epoch 161][10/66] loss=0.787767 avg=0.811670 VRAM=39.0GiB | 32.2% done | ETA(epoch): 1076s + [Epoch 161][20/66] loss=0.734807 avg=0.836958 VRAM=38.8GiB | 32.3% done | ETA(epoch): 884s + [Epoch 161][30/66] loss=0.798653 avg=0.851808 VRAM=39.0GiB | 32.3% done | ETA(epoch): 692s + [Epoch 161][40/66] loss=0.908150 avg=0.844593 VRAM=38.8GiB | 32.3% done | ETA(epoch): 500s + [Epoch 161][50/66] loss=0.965235 avg=0.849889 VRAM=39.0GiB | 32.4% done | ETA(epoch): 308s + [Epoch 161][60/66] loss=0.809051 avg=0.843729 VRAM=38.8GiB | 32.4% done | ETA(epoch): 115s + Train loss: 0.840630 (1270.3s) ETA: 7810min + Val loss: 0.871054 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9924 t_0.4-0.6=0.8470 t_0.6-0.8=0.7348 t_0.8-1.0=0.6344] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 162/499 (32% done) --- + [Epoch 162][10/66] loss=0.825031 avg=0.864330 VRAM=39.0GiB | 32.4% done | ETA(epoch): 1076s + [Epoch 162][20/66] loss=0.871964 avg=0.849950 VRAM=38.8GiB | 32.5% done | ETA(epoch): 884s + [Epoch 162][30/66] loss=0.901309 avg=0.847335 VRAM=39.0GiB | 32.5% done | ETA(epoch): 691s + [Epoch 162][40/66] loss=0.828297 avg=0.851237 VRAM=38.8GiB | 32.5% done | ETA(epoch): 500s + [Epoch 162][50/66] loss=0.938394 avg=0.858684 VRAM=39.0GiB | 32.6% done | ETA(epoch): 307s + [Epoch 162][60/66] loss=0.795560 avg=0.858514 VRAM=38.8GiB | 32.6% done | ETA(epoch): 115s + Train loss: 0.855613 (1268.4s) ETA: 7786min + Val loss: 0.805565 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0083 t_0.4-0.6=0.8404 t_0.6-0.8=0.7178 t_0.8-1.0=0.6116] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0162 (BEST) + Deleted old checkpoint: checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 163/499 (33% done) --- + [Epoch 163][10/66] loss=0.683564 avg=0.834790 VRAM=39.0GiB | 32.6% done | ETA(epoch): 1075s + [Epoch 163][20/66] loss=0.867060 avg=0.870652 VRAM=38.8GiB | 32.7% done | ETA(epoch): 884s + [Epoch 163][30/66] loss=0.767124 avg=0.864834 VRAM=39.0GiB | 32.7% done | ETA(epoch): 691s + [Epoch 163][40/66] loss=0.867315 avg=0.863102 VRAM=38.8GiB | 32.7% done | ETA(epoch): 501s + [Epoch 163][50/66] loss=0.873857 avg=0.865663 VRAM=39.0GiB | 32.8% done | ETA(epoch): 308s + [Epoch 163][60/66] loss=1.004438 avg=0.867504 VRAM=38.8GiB | 32.8% done | ETA(epoch): 115s + Train loss: 0.863280 (1270.0s) ETA: 7762min + Val loss: 0.871441 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9634 t_0.4-0.6=0.8740 t_0.6-0.8=0.7121 t_0.8-1.0=0.6155] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 164/499 (33% done) --- + [Epoch 164][10/66] loss=0.943348 avg=0.919125 VRAM=39.0GiB | 32.8% done | ETA(epoch): 1077s + [Epoch 164][20/66] loss=0.803983 avg=0.878211 VRAM=38.8GiB | 32.9% done | ETA(epoch): 884s + [Epoch 164][30/66] loss=0.815765 avg=0.861440 VRAM=39.0GiB | 32.9% done | ETA(epoch): 692s + [Epoch 164][40/66] loss=0.871400 avg=0.866414 VRAM=38.8GiB | 32.9% done | ETA(epoch): 500s + [Epoch 164][50/66] loss=0.780240 avg=0.867038 VRAM=39.0GiB | 33.0% done | ETA(epoch): 308s + [Epoch 164][60/66] loss=0.732319 avg=0.859220 VRAM=38.8GiB | 33.0% done | ETA(epoch): 115s + Train loss: 0.856169 (1269.2s) ETA: 7738min + Val loss: 0.886499 [t_0.0-0.2=1.0774 t_0.2-0.4=0.9798 t_0.4-0.6=0.8447 t_0.6-0.8=0.7112 t_0.8-1.0=0.6491] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 165/499 (33% done) --- + [Epoch 165][10/66] loss=0.835249 avg=0.860207 VRAM=39.0GiB | 33.0% done | ETA(epoch): 1077s + [Epoch 165][20/66] loss=0.982713 avg=0.869161 VRAM=38.8GiB | 33.1% done | ETA(epoch): 890s + [Epoch 165][30/66] loss=0.901620 avg=0.867624 VRAM=39.0GiB | 33.1% done | ETA(epoch): 695s + [Epoch 165][40/66] loss=0.921590 avg=0.868931 VRAM=38.8GiB | 33.1% done | ETA(epoch): 502s + [Epoch 165][50/66] loss=0.803137 avg=0.866831 VRAM=39.0GiB | 33.2% done | ETA(epoch): 309s + [Epoch 165][60/66] loss=0.859309 avg=0.865588 VRAM=38.8GiB | 33.2% done | ETA(epoch): 116s + Train loss: 0.867126 (1272.4s) ETA: 7715min + Val loss: 0.859728 [t_0.0-0.2=1.0707 t_0.2-0.4=1.0416 t_0.4-0.6=0.8352 t_0.6-0.8=0.6978 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0165 +[MEM @ epoch 165 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 166/499 (33% done) --- + [Epoch 166][10/66] loss=0.767566 avg=0.828931 VRAM=39.0GiB | 33.2% done | ETA(epoch): 1087s + [Epoch 166][20/66] loss=0.904547 avg=0.847124 VRAM=38.8GiB | 33.3% done | ETA(epoch): 889s + [Epoch 166][30/66] loss=0.762412 avg=0.855271 VRAM=39.0GiB | 33.3% done | ETA(epoch): 695s + [Epoch 166][40/66] loss=0.836633 avg=0.848615 VRAM=38.8GiB | 33.3% done | ETA(epoch): 501s + [Epoch 166][50/66] loss=0.832738 avg=0.842615 VRAM=39.0GiB | 33.4% done | ETA(epoch): 308s + [Epoch 166][60/66] loss=0.969806 avg=0.849201 VRAM=38.8GiB | 33.4% done | ETA(epoch): 116s + Train loss: 0.847850 (1271.5s) ETA: 7691min + Val loss: 0.819631 [t_0.0-0.2=1.0930 t_0.2-0.4=0.9668 t_0.4-0.6=0.8323 t_0.6-0.8=0.7019 t_0.8-1.0=0.6420] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 167/499 (33% done) --- + [Epoch 167][10/66] loss=0.830657 avg=0.861318 VRAM=39.0GiB | 33.4% done | ETA(epoch): 1076s + [Epoch 167][20/66] loss=0.729231 avg=0.860838 VRAM=38.8GiB | 33.5% done | ETA(epoch): 884s + [Epoch 167][30/66] loss=0.862909 avg=0.846912 VRAM=39.0GiB | 33.5% done | ETA(epoch): 692s + [Epoch 167][40/66] loss=0.919214 avg=0.839179 VRAM=38.8GiB | 33.5% done | ETA(epoch): 500s + [Epoch 167][50/66] loss=0.843338 avg=0.839133 VRAM=39.0GiB | 33.6% done | ETA(epoch): 308s + [Epoch 167][60/66] loss=0.845708 avg=0.836752 VRAM=38.8GiB | 33.6% done | ETA(epoch): 115s + Train loss: 0.840165 (1273.3s) ETA: 7667min + Val loss: 0.830066 [t_0.0-0.2=1.0645 t_0.2-0.4=1.0300 t_0.4-0.6=0.8323 t_0.6-0.8=0.7027 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 168/499 (34% done) --- + [Epoch 168][10/66] loss=0.884269 avg=0.868077 VRAM=39.0GiB | 33.6% done | ETA(epoch): 1077s + [Epoch 168][20/66] loss=0.884485 avg=0.851659 VRAM=38.8GiB | 33.7% done | ETA(epoch): 884s + [Epoch 168][30/66] loss=0.956552 avg=0.870023 VRAM=39.0GiB | 33.7% done | ETA(epoch): 692s + [Epoch 168][40/66] loss=0.802923 avg=0.861100 VRAM=38.8GiB | 33.7% done | ETA(epoch): 500s + [Epoch 168][50/66] loss=0.864539 avg=0.858973 VRAM=39.0GiB | 33.8% done | ETA(epoch): 308s + [Epoch 168][60/66] loss=0.688593 avg=0.856090 VRAM=38.8GiB | 33.8% done | ETA(epoch): 116s + Train loss: 0.851228 (1271.5s) ETA: 7643min + Val loss: 0.818639 [t_0.0-0.2=1.0601 t_0.2-0.4=0.9996 t_0.4-0.6=0.8466 t_0.6-0.8=0.7241 t_0.8-1.0=0.6413] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 169/499 (34% done) --- + [Epoch 169][10/66] loss=0.785288 avg=0.829481 VRAM=39.0GiB | 33.8% done | ETA(epoch): 1076s + [Epoch 169][20/66] loss=0.981449 avg=0.848623 VRAM=38.8GiB | 33.9% done | ETA(epoch): 884s + [Epoch 169][30/66] loss=0.895812 avg=0.858863 VRAM=39.0GiB | 33.9% done | ETA(epoch): 692s + [Epoch 169][40/66] loss=0.761983 avg=0.853024 VRAM=38.8GiB | 33.9% done | ETA(epoch): 500s + [Epoch 169][50/66] loss=0.822774 avg=0.861402 VRAM=39.0GiB | 34.0% done | ETA(epoch): 307s + [Epoch 169][60/66] loss=0.853471 avg=0.864560 VRAM=38.8GiB | 34.0% done | ETA(epoch): 115s + Train loss: 0.862341 (1268.3s) ETA: 7620min + Val loss: 0.865478 [t_0.0-0.2=1.0764 t_0.2-0.4=1.0208 t_0.4-0.6=0.8337 t_0.6-0.8=0.6731 t_0.8-1.0=0.6393] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 170/499 (34% done) --- + [Epoch 170][10/66] loss=0.902630 avg=0.822128 VRAM=39.0GiB | 34.0% done | ETA(epoch): 1077s + [Epoch 170][20/66] loss=0.925888 avg=0.838632 VRAM=38.8GiB | 34.1% done | ETA(epoch): 885s + [Epoch 170][30/66] loss=0.913030 avg=0.841386 VRAM=39.0GiB | 34.1% done | ETA(epoch): 693s + [Epoch 170][40/66] loss=0.960768 avg=0.849441 VRAM=38.8GiB | 34.1% done | ETA(epoch): 501s + [Epoch 170][50/66] loss=0.871844 avg=0.845765 VRAM=39.0GiB | 34.2% done | ETA(epoch): 308s + [Epoch 170][60/66] loss=0.916560 avg=0.849619 VRAM=38.8GiB | 34.2% done | ETA(epoch): 115s + Train loss: 0.848564 (1270.2s) ETA: 7596min + Val loss: 0.842578 [t_0.0-0.2=1.0706 t_0.2-0.4=0.9996 t_0.4-0.6=0.8622 t_0.6-0.8=0.6881 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 171/499 (34% done) --- + [Epoch 171][10/66] loss=0.823425 avg=0.846609 VRAM=39.0GiB | 34.2% done | ETA(epoch): 1076s + [Epoch 171][20/66] loss=0.760628 avg=0.844013 VRAM=38.8GiB | 34.3% done | ETA(epoch): 884s + [Epoch 171][30/66] loss=0.788438 avg=0.847093 VRAM=39.0GiB | 34.3% done | ETA(epoch): 693s + [Epoch 171][40/66] loss=0.830153 avg=0.850119 VRAM=38.8GiB | 34.3% done | ETA(epoch): 500s + [Epoch 171][50/66] loss=0.834098 avg=0.853778 VRAM=39.0GiB | 34.4% done | ETA(epoch): 308s + [Epoch 171][60/66] loss=0.745118 avg=0.854804 VRAM=38.8GiB | 34.4% done | ETA(epoch): 115s + Train loss: 0.852438 (1269.6s) ETA: 7572min + Val loss: 0.819408 [t_0.0-0.2=1.0720 t_0.2-0.4=0.9970 t_0.4-0.6=0.8500 t_0.6-0.8=0.7047 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 172/499 (34% done) --- + [Epoch 172][10/66] loss=0.654709 avg=0.859137 VRAM=39.0GiB | 34.4% done | ETA(epoch): 1078s + [Epoch 172][20/66] loss=0.808794 avg=0.852875 VRAM=38.8GiB | 34.5% done | ETA(epoch): 885s + [Epoch 172][30/66] loss=0.793860 avg=0.847965 VRAM=39.0GiB | 34.5% done | ETA(epoch): 692s + [Epoch 172][40/66] loss=0.865810 avg=0.845870 VRAM=38.8GiB | 34.5% done | ETA(epoch): 500s + [Epoch 172][50/66] loss=0.875940 avg=0.845940 VRAM=39.0GiB | 34.6% done | ETA(epoch): 308s + [Epoch 172][60/66] loss=0.875827 avg=0.852470 VRAM=38.8GiB | 34.6% done | ETA(epoch): 115s + Train loss: 0.853859 (1269.3s) ETA: 7548min + Val loss: 0.832808 [t_0.0-0.2=1.0497 t_0.2-0.4=1.0133 t_0.4-0.6=0.8527 t_0.6-0.8=0.7218 t_0.8-1.0=0.6461] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 173/499 (35% done) --- + [Epoch 173][10/66] loss=0.941034 avg=0.889270 VRAM=39.0GiB | 34.6% done | ETA(epoch): 1077s + [Epoch 173][20/66] loss=0.818277 avg=0.877378 VRAM=38.8GiB | 34.7% done | ETA(epoch): 885s + [Epoch 173][30/66] loss=0.907359 avg=0.865237 VRAM=39.0GiB | 34.7% done | ETA(epoch): 692s + [Epoch 173][40/66] loss=0.823930 avg=0.856992 VRAM=38.8GiB | 34.7% done | ETA(epoch): 500s + [Epoch 173][50/66] loss=0.739816 avg=0.852538 VRAM=39.0GiB | 34.8% done | ETA(epoch): 308s + [Epoch 173][60/66] loss=0.912637 avg=0.847196 VRAM=38.8GiB | 34.8% done | ETA(epoch): 115s + Train loss: 0.847419 (1268.7s) ETA: 7525min + Val loss: 0.867848 [t_0.0-0.2=1.0898 t_0.2-0.4=0.9904 t_0.4-0.6=0.8367 t_0.6-0.8=0.6898 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 174/499 (35% done) --- + [Epoch 174][10/66] loss=0.786178 avg=0.795197 VRAM=39.0GiB | 34.8% done | ETA(epoch): 1075s + [Epoch 174][20/66] loss=0.923029 avg=0.854261 VRAM=38.8GiB | 34.9% done | ETA(epoch): 883s + [Epoch 174][30/66] loss=0.889874 avg=0.854958 VRAM=39.0GiB | 34.9% done | ETA(epoch): 691s + [Epoch 174][40/66] loss=0.866419 avg=0.857476 VRAM=38.8GiB | 34.9% done | ETA(epoch): 500s + [Epoch 174][50/66] loss=0.787549 avg=0.858859 VRAM=39.0GiB | 35.0% done | ETA(epoch): 307s + [Epoch 174][60/66] loss=0.804347 avg=0.860172 VRAM=38.8GiB | 35.0% done | ETA(epoch): 115s + Train loss: 0.855491 (1268.2s) ETA: 7501min + Val loss: 0.858872 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9850 t_0.4-0.6=0.8523 t_0.6-0.8=0.7162 t_0.8-1.0=0.6241] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 175/499 (35% done) --- + [Epoch 175][10/66] loss=0.823657 avg=0.832512 VRAM=39.0GiB | 35.0% done | ETA(epoch): 1076s + [Epoch 175][20/66] loss=0.897491 avg=0.857251 VRAM=38.8GiB | 35.1% done | ETA(epoch): 884s + [Epoch 175][30/66] loss=0.936524 avg=0.862543 VRAM=39.0GiB | 35.1% done | ETA(epoch): 692s + [Epoch 175][40/66] loss=0.853685 avg=0.854428 VRAM=38.8GiB | 35.1% done | ETA(epoch): 500s + [Epoch 175][50/66] loss=0.922435 avg=0.857440 VRAM=39.0GiB | 35.2% done | ETA(epoch): 307s + [Epoch 175][60/66] loss=0.830533 avg=0.858595 VRAM=38.8GiB | 35.2% done | ETA(epoch): 115s + Train loss: 0.855689 (1270.3s) ETA: 7477min + Val loss: 0.851020 [t_0.0-0.2=1.0753 t_0.2-0.4=0.9574 t_0.4-0.6=0.8563 t_0.6-0.8=0.6933 t_0.8-1.0=0.6311] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 176/499 (35% done) --- + [Epoch 176][10/66] loss=0.768222 avg=0.863485 VRAM=39.0GiB | 35.2% done | ETA(epoch): 1075s + [Epoch 176][20/66] loss=0.826119 avg=0.864448 VRAM=38.8GiB | 35.3% done | ETA(epoch): 884s + [Epoch 176][30/66] loss=0.779310 avg=0.857873 VRAM=39.0GiB | 35.3% done | ETA(epoch): 692s + [Epoch 176][40/66] loss=0.837469 avg=0.853609 VRAM=38.8GiB | 35.3% done | ETA(epoch): 500s + [Epoch 176][50/66] loss=0.764197 avg=0.851818 VRAM=39.0GiB | 35.4% done | ETA(epoch): 308s + [Epoch 176][60/66] loss=0.786909 avg=0.845113 VRAM=38.8GiB | 35.4% done | ETA(epoch): 116s + Train loss: 0.843358 (1270.7s) ETA: 7453min + Val loss: 0.868543 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9972 t_0.4-0.6=0.8094 t_0.6-0.8=0.7275 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 177/499 (35% done) --- + [Epoch 177][10/66] loss=0.799252 avg=0.811367 VRAM=39.0GiB | 35.4% done | ETA(epoch): 1075s + [Epoch 177][20/66] loss=0.992142 avg=0.839761 VRAM=38.8GiB | 35.5% done | ETA(epoch): 883s + [Epoch 177][30/66] loss=0.941617 avg=0.851512 VRAM=39.0GiB | 35.5% done | ETA(epoch): 691s + [Epoch 177][40/66] loss=0.802229 avg=0.847656 VRAM=38.8GiB | 35.5% done | ETA(epoch): 499s + [Epoch 177][50/66] loss=0.850708 avg=0.845236 VRAM=39.0GiB | 35.6% done | ETA(epoch): 308s + [Epoch 177][60/66] loss=0.966938 avg=0.848972 VRAM=38.8GiB | 35.6% done | ETA(epoch): 115s + Train loss: 0.848676 (1269.7s) ETA: 7430min + Val loss: 0.835689 [t_0.0-0.2=1.0608 t_0.2-0.4=1.0007 t_0.4-0.6=0.8301 t_0.6-0.8=0.7056 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 178/499 (36% done) --- + [Epoch 178][10/66] loss=0.828885 avg=0.845412 VRAM=39.0GiB | 35.6% done | ETA(epoch): 1076s + [Epoch 178][20/66] loss=0.838593 avg=0.847782 VRAM=38.8GiB | 35.7% done | ETA(epoch): 884s + [Epoch 178][30/66] loss=0.891331 avg=0.843060 VRAM=39.0GiB | 35.7% done | ETA(epoch): 692s + [Epoch 178][40/66] loss=0.908854 avg=0.851114 VRAM=38.8GiB | 35.7% done | ETA(epoch): 505s + [Epoch 178][50/66] loss=0.815574 avg=0.847783 VRAM=39.0GiB | 35.8% done | ETA(epoch): 310s + [Epoch 178][60/66] loss=0.860053 avg=0.848523 VRAM=38.8GiB | 35.8% done | ETA(epoch): 116s + Train loss: 0.848568 (1276.3s) ETA: 7406min + Val loss: 0.844524 [t_0.0-0.2=1.0760 t_0.2-0.4=1.0323 t_0.4-0.6=0.8372 t_0.6-0.8=0.7232 t_0.8-1.0=0.6213] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 179/499 (36% done) --- + [Epoch 179][10/66] loss=0.810084 avg=0.839532 VRAM=39.0GiB | 35.8% done | ETA(epoch): 1075s + [Epoch 179][20/66] loss=0.883819 avg=0.845878 VRAM=38.8GiB | 35.9% done | ETA(epoch): 883s + [Epoch 179][30/66] loss=0.834348 avg=0.853479 VRAM=39.0GiB | 35.9% done | ETA(epoch): 691s + [Epoch 179][40/66] loss=0.856148 avg=0.854946 VRAM=38.8GiB | 35.9% done | ETA(epoch): 499s + [Epoch 179][50/66] loss=0.989195 avg=0.855540 VRAM=39.0GiB | 36.0% done | ETA(epoch): 307s + [Epoch 179][60/66] loss=0.722698 avg=0.847791 VRAM=38.8GiB | 36.0% done | ETA(epoch): 115s + Train loss: 0.851186 (1268.3s) ETA: 7383min + Val loss: 0.818509 [t_0.0-0.2=1.0675 t_0.2-0.4=0.9961 t_0.4-0.6=0.8313 t_0.6-0.8=0.7135 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 180/499 (36% done) --- + [Epoch 180][10/66] loss=0.918855 avg=0.870921 VRAM=39.0GiB | 36.0% done | ETA(epoch): 1076s + [Epoch 180][20/66] loss=0.833733 avg=0.862324 VRAM=38.8GiB | 36.1% done | ETA(epoch): 885s + [Epoch 180][30/66] loss=0.821700 avg=0.863692 VRAM=39.0GiB | 36.1% done | ETA(epoch): 692s + [Epoch 180][40/66] loss=0.882243 avg=0.848644 VRAM=38.8GiB | 36.1% done | ETA(epoch): 500s + [Epoch 180][50/66] loss=0.791921 avg=0.854804 VRAM=39.0GiB | 36.2% done | ETA(epoch): 308s + [Epoch 180][60/66] loss=0.778879 avg=0.850983 VRAM=38.8GiB | 36.2% done | ETA(epoch): 115s + Train loss: 0.847160 (1269.1s) ETA: 7359min + Val loss: 0.828829 [t_0.0-0.2=1.0607 t_0.2-0.4=0.9782 t_0.4-0.6=0.8126 t_0.6-0.8=0.6723 t_0.8-1.0=0.6299] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 181/499 (36% done) --- + [Epoch 181][10/66] loss=0.811618 avg=0.836261 VRAM=39.0GiB | 36.2% done | ETA(epoch): 1079s + [Epoch 181][20/66] loss=0.702888 avg=0.814667 VRAM=38.8GiB | 36.3% done | ETA(epoch): 885s + [Epoch 181][30/66] loss=0.886011 avg=0.820017 VRAM=39.0GiB | 36.3% done | ETA(epoch): 692s + [Epoch 181][40/66] loss=0.806698 avg=0.827713 VRAM=38.8GiB | 36.3% done | ETA(epoch): 500s + [Epoch 181][50/66] loss=0.994534 avg=0.843235 VRAM=39.0GiB | 36.4% done | ETA(epoch): 308s + [Epoch 181][60/66] loss=0.758120 avg=0.841785 VRAM=38.8GiB | 36.4% done | ETA(epoch): 115s + Train loss: 0.839899 (1269.3s) ETA: 7335min + Val loss: 0.810073 [t_0.0-0.2=1.0747 t_0.2-0.4=1.0084 t_0.4-0.6=0.8194 t_0.6-0.8=0.7171 t_0.8-1.0=0.6304] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 182/499 (36% done) --- + [Epoch 182][10/66] loss=0.922521 avg=0.881089 VRAM=39.0GiB | 36.4% done | ETA(epoch): 1076s + [Epoch 182][20/66] loss=0.715261 avg=0.864951 VRAM=38.8GiB | 36.5% done | ETA(epoch): 884s + [Epoch 182][30/66] loss=0.806889 avg=0.833673 VRAM=39.0GiB | 36.5% done | ETA(epoch): 692s + [Epoch 182][40/66] loss=0.766536 avg=0.838789 VRAM=38.8GiB | 36.5% done | ETA(epoch): 500s + [Epoch 182][50/66] loss=0.775146 avg=0.832607 VRAM=39.0GiB | 36.6% done | ETA(epoch): 307s + [Epoch 182][60/66] loss=0.756486 avg=0.822727 VRAM=38.8GiB | 36.6% done | ETA(epoch): 115s + Train loss: 0.822736 (1286.3s) ETA: 7312min + Val loss: 0.845265 [t_0.0-0.2=1.0774 t_0.2-0.4=0.9877 t_0.4-0.6=0.8591 t_0.6-0.8=0.7209 t_0.8-1.0=0.6215] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 183/499 (37% done) --- + [Epoch 183][10/66] loss=0.818104 avg=0.874000 VRAM=39.0GiB | 36.6% done | ETA(epoch): 1075s + [Epoch 183][20/66] loss=0.892491 avg=0.857502 VRAM=38.8GiB | 36.7% done | ETA(epoch): 884s + [Epoch 183][30/66] loss=0.926232 avg=0.860769 VRAM=39.0GiB | 36.7% done | ETA(epoch): 692s + [Epoch 183][40/66] loss=0.918208 avg=0.863711 VRAM=38.8GiB | 36.7% done | ETA(epoch): 500s + [Epoch 183][50/66] loss=0.885428 avg=0.855854 VRAM=39.0GiB | 36.8% done | ETA(epoch): 308s + [Epoch 183][60/66] loss=0.858131 avg=0.851904 VRAM=38.8GiB | 36.8% done | ETA(epoch): 116s + Train loss: 0.856391 (1272.5s) ETA: 7289min + Val loss: 0.817464 [t_0.0-0.2=1.0292 t_0.2-0.4=1.0115 t_0.4-0.6=0.8579 t_0.6-0.8=0.7056 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 184/499 (37% done) --- + [Epoch 184][10/66] loss=0.853351 avg=0.837900 VRAM=39.0GiB | 36.8% done | ETA(epoch): 1077s + [Epoch 184][20/66] loss=0.821292 avg=0.836836 VRAM=38.8GiB | 36.9% done | ETA(epoch): 884s + [Epoch 184][30/66] loss=0.920047 avg=0.840669 VRAM=39.0GiB | 36.9% done | ETA(epoch): 692s + [Epoch 184][40/66] loss=0.890025 avg=0.839791 VRAM=38.8GiB | 36.9% done | ETA(epoch): 500s + [Epoch 184][50/66] loss=0.815083 avg=0.843914 VRAM=39.0GiB | 37.0% done | ETA(epoch): 308s + [Epoch 184][60/66] loss=0.902734 avg=0.853245 VRAM=38.8GiB | 37.0% done | ETA(epoch): 115s + Train loss: 0.854620 (1268.6s) ETA: 7265min + Val loss: 0.851970 [t_0.0-0.2=1.0683 t_0.2-0.4=1.0126 t_0.4-0.6=0.8147 t_0.6-0.8=0.7221 t_0.8-1.0=0.6234] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 185/499 (37% done) --- + [Epoch 185][10/66] loss=0.666568 avg=0.840407 VRAM=39.0GiB | 37.0% done | ETA(epoch): 1074s + [Epoch 185][20/66] loss=0.731730 avg=0.842259 VRAM=38.8GiB | 37.1% done | ETA(epoch): 885s + [Epoch 185][30/66] loss=0.736242 avg=0.839962 VRAM=39.0GiB | 37.1% done | ETA(epoch): 692s + [Epoch 185][40/66] loss=0.895081 avg=0.831321 VRAM=38.8GiB | 37.1% done | ETA(epoch): 500s + [Epoch 185][50/66] loss=0.837639 avg=0.833526 VRAM=39.0GiB | 37.2% done | ETA(epoch): 308s + [Epoch 185][60/66] loss=0.882776 avg=0.842970 VRAM=38.8GiB | 37.2% done | ETA(epoch): 115s + Train loss: 0.846934 (1268.6s) ETA: 7241min + Val loss: 0.822665 [t_0.0-0.2=1.0823 t_0.2-0.4=0.9857 t_0.4-0.6=0.8435 t_0.6-0.8=0.7002 t_0.8-1.0=0.6298] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 186/499 (37% done) --- + [Epoch 186][10/66] loss=0.916444 avg=0.854683 VRAM=39.0GiB | 37.2% done | ETA(epoch): 1078s + [Epoch 186][20/66] loss=0.771816 avg=0.834493 VRAM=38.8GiB | 37.3% done | ETA(epoch): 885s + [Epoch 186][30/66] loss=0.850320 avg=0.838636 VRAM=39.0GiB | 37.3% done | ETA(epoch): 695s + [Epoch 186][40/66] loss=0.835667 avg=0.843155 VRAM=38.8GiB | 37.3% done | ETA(epoch): 502s + [Epoch 186][50/66] loss=0.917323 avg=0.845721 VRAM=39.0GiB | 37.4% done | ETA(epoch): 308s + [Epoch 186][60/66] loss=0.712437 avg=0.843915 VRAM=38.8GiB | 37.4% done | ETA(epoch): 116s + Train loss: 0.844686 (1271.1s) ETA: 7218min + Val loss: 0.859124 [t_0.0-0.2=1.0662 t_0.2-0.4=0.9810 t_0.4-0.6=0.8154 t_0.6-0.8=0.6992 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 18.7/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 187/499 (37% done) --- + [Epoch 187][10/66] loss=0.842442 avg=0.836500 VRAM=39.0GiB | 37.4% done | ETA(epoch): 1076s + [Epoch 187][20/66] loss=0.861052 avg=0.841568 VRAM=38.8GiB | 37.5% done | ETA(epoch): 884s + [Epoch 187][30/66] loss=0.786211 avg=0.828098 VRAM=39.0GiB | 37.5% done | ETA(epoch): 692s + [Epoch 187][40/66] loss=0.845443 avg=0.830312 VRAM=38.8GiB | 37.5% done | ETA(epoch): 500s + [Epoch 187][50/66] loss=0.947152 avg=0.837877 VRAM=39.0GiB | 37.6% done | ETA(epoch): 307s + [Epoch 187][60/66] loss=0.733813 avg=0.833451 VRAM=38.8GiB | 37.6% done | ETA(epoch): 115s + Train loss: 0.829431 (1268.5s) ETA: 7194min + Val loss: 0.843496 [t_0.0-0.2=1.0643 t_0.2-0.4=0.9839 t_0.4-0.6=0.8512 t_0.6-0.8=0.6767 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 188/499 (38% done) --- + [Epoch 188][10/66] loss=0.977309 avg=0.881047 VRAM=39.0GiB | 37.6% done | ETA(epoch): 1075s + [Epoch 188][20/66] loss=0.928108 avg=0.863560 VRAM=38.8GiB | 37.7% done | ETA(epoch): 883s + [Epoch 188][30/66] loss=0.886118 avg=0.852782 VRAM=39.0GiB | 37.7% done | ETA(epoch): 691s + [Epoch 188][40/66] loss=0.894279 avg=0.856570 VRAM=38.8GiB | 37.7% done | ETA(epoch): 499s + [Epoch 188][50/66] loss=0.969330 avg=0.856296 VRAM=39.0GiB | 37.8% done | ETA(epoch): 307s + [Epoch 188][60/66] loss=0.781460 avg=0.853198 VRAM=38.8GiB | 37.8% done | ETA(epoch): 115s + Train loss: 0.848352 (1268.2s) ETA: 7171min + Val loss: 0.805406 [t_0.0-0.2=1.0683 t_0.2-0.4=0.9473 t_0.4-0.6=0.8394 t_0.6-0.8=0.7007 t_0.8-1.0=0.6252] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0188 (BEST) + Deleted old checkpoint: checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 189/499 (38% done) --- + [Epoch 189][10/66] loss=0.864910 avg=0.849139 VRAM=39.0GiB | 37.8% done | ETA(epoch): 1075s + [Epoch 189][20/66] loss=0.878867 avg=0.848915 VRAM=38.8GiB | 37.9% done | ETA(epoch): 884s + [Epoch 189][30/66] loss=0.787418 avg=0.840587 VRAM=39.0GiB | 37.9% done | ETA(epoch): 692s + [Epoch 189][40/66] loss=0.855426 avg=0.837976 VRAM=38.8GiB | 37.9% done | ETA(epoch): 500s + [Epoch 189][50/66] loss=0.753021 avg=0.837618 VRAM=39.0GiB | 38.0% done | ETA(epoch): 308s + [Epoch 189][60/66] loss=0.810959 avg=0.836901 VRAM=38.8GiB | 38.0% done | ETA(epoch): 115s + Train loss: 0.836084 (1269.2s) ETA: 7147min + Val loss: 0.884837 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9872 t_0.4-0.6=0.8612 t_0.6-0.8=0.6924 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 190/499 (38% done) --- + [Epoch 190][10/66] loss=0.782788 avg=0.840810 VRAM=39.0GiB | 38.0% done | ETA(epoch): 1075s + [Epoch 190][20/66] loss=0.896518 avg=0.850262 VRAM=38.8GiB | 38.1% done | ETA(epoch): 883s + [Epoch 190][30/66] loss=0.843134 avg=0.852873 VRAM=39.0GiB | 38.1% done | ETA(epoch): 692s + [Epoch 190][40/66] loss=0.763332 avg=0.851478 VRAM=38.8GiB | 38.1% done | ETA(epoch): 500s + [Epoch 190][50/66] loss=0.994692 avg=0.855693 VRAM=39.0GiB | 38.2% done | ETA(epoch): 308s + [Epoch 190][60/66] loss=0.987656 avg=0.853828 VRAM=38.8GiB | 38.2% done | ETA(epoch): 116s + Train loss: 0.855561 (1270.7s) ETA: 7123min + Val loss: 0.836215 [t_0.0-0.2=1.0805 t_0.2-0.4=0.9788 t_0.4-0.6=0.8239 t_0.6-0.8=0.7164 t_0.8-1.0=0.6224] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 191/499 (38% done) --- + [Epoch 191][10/66] loss=0.865930 avg=0.866546 VRAM=39.0GiB | 38.2% done | ETA(epoch): 1074s + [Epoch 191][20/66] loss=0.927099 avg=0.860972 VRAM=38.8GiB | 38.3% done | ETA(epoch): 883s + [Epoch 191][30/66] loss=0.789475 avg=0.864764 VRAM=39.0GiB | 38.3% done | ETA(epoch): 692s + [Epoch 191][40/66] loss=0.849377 avg=0.858486 VRAM=38.8GiB | 38.3% done | ETA(epoch): 500s + [Epoch 191][50/66] loss=0.827272 avg=0.858127 VRAM=39.0GiB | 38.4% done | ETA(epoch): 307s + [Epoch 191][60/66] loss=0.789265 avg=0.847460 VRAM=38.8GiB | 38.4% done | ETA(epoch): 115s + Train loss: 0.844815 (1268.3s) ETA: 7100min + Val loss: 0.838562 [t_0.0-0.2=1.0738 t_0.2-0.4=1.0002 t_0.4-0.6=0.8176 t_0.6-0.8=0.7142 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0191 +[MEM @ epoch 191 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 192/499 (38% done) --- + [Epoch 192][10/66] loss=0.777885 avg=0.855372 VRAM=39.0GiB | 38.4% done | ETA(epoch): 1079s + [Epoch 192][20/66] loss=0.856058 avg=0.861999 VRAM=38.8GiB | 38.5% done | ETA(epoch): 888s + [Epoch 192][30/66] loss=0.892754 avg=0.847929 VRAM=39.0GiB | 38.5% done | ETA(epoch): 695s + [Epoch 192][40/66] loss=0.777278 avg=0.850519 VRAM=38.8GiB | 38.5% done | ETA(epoch): 502s + [Epoch 192][50/66] loss=0.877074 avg=0.851875 VRAM=39.0GiB | 38.6% done | ETA(epoch): 309s + [Epoch 192][60/66] loss=0.807752 avg=0.848853 VRAM=38.8GiB | 38.6% done | ETA(epoch): 116s + Train loss: 0.846537 (1273.9s) ETA: 7076min + Val loss: 0.831045 [t_0.0-0.2=1.0737 t_0.2-0.4=0.9956 t_0.4-0.6=0.8370 t_0.6-0.8=0.7357 t_0.8-1.0=0.6353] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 193/499 (39% done) --- + [Epoch 193][10/66] loss=0.832400 avg=0.843345 VRAM=39.0GiB | 38.6% done | ETA(epoch): 1074s + [Epoch 193][20/66] loss=0.693555 avg=0.848137 VRAM=38.8GiB | 38.7% done | ETA(epoch): 883s + [Epoch 193][30/66] loss=0.802384 avg=0.849678 VRAM=39.0GiB | 38.7% done | ETA(epoch): 691s + [Epoch 193][40/66] loss=0.847911 avg=0.844983 VRAM=38.8GiB | 38.7% done | ETA(epoch): 504s + [Epoch 193][50/66] loss=0.873887 avg=0.846331 VRAM=39.0GiB | 38.8% done | ETA(epoch): 309s + [Epoch 193][60/66] loss=0.897982 avg=0.846431 VRAM=38.8GiB | 38.8% done | ETA(epoch): 116s + Train loss: 0.843636 (1274.4s) ETA: 7053min + Val loss: 0.871973 [t_0.0-0.2=1.0653 t_0.2-0.4=0.9840 t_0.4-0.6=0.8553 t_0.6-0.8=0.7099 t_0.8-1.0=0.6316] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 18.4/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 194/499 (39% done) --- + [Epoch 194][10/66] loss=0.726037 avg=0.859180 VRAM=39.0GiB | 38.8% done | ETA(epoch): 1075s + [Epoch 194][20/66] loss=0.875758 avg=0.872762 VRAM=38.8GiB | 38.9% done | ETA(epoch): 884s + [Epoch 194][30/66] loss=0.804173 avg=0.863514 VRAM=39.0GiB | 38.9% done | ETA(epoch): 692s + [Epoch 194][40/66] loss=0.744479 avg=0.861595 VRAM=38.8GiB | 38.9% done | ETA(epoch): 500s + [Epoch 194][50/66] loss=0.805191 avg=0.857451 VRAM=39.0GiB | 39.0% done | ETA(epoch): 307s + [Epoch 194][60/66] loss=0.842891 avg=0.857823 VRAM=38.8GiB | 39.0% done | ETA(epoch): 115s + Train loss: 0.853907 (1268.7s) ETA: 7029min + Val loss: 0.822062 [t_0.0-0.2=1.0451 t_0.2-0.4=0.9640 t_0.4-0.6=0.8507 t_0.6-0.8=0.6997 t_0.8-1.0=0.6042] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 18.6/188.5 GiB (9.9%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 195/499 (39% done) --- + [Epoch 195][10/66] loss=0.790558 avg=0.873794 VRAM=39.0GiB | 39.0% done | ETA(epoch): 1077s + [Epoch 195][20/66] loss=0.755216 avg=0.851742 VRAM=38.8GiB | 39.1% done | ETA(epoch): 885s + [Epoch 195][30/66] loss=0.895199 avg=0.858055 VRAM=39.0GiB | 39.1% done | ETA(epoch): 692s + [Epoch 195][40/66] loss=0.859901 avg=0.848336 VRAM=38.8GiB | 39.1% done | ETA(epoch): 500s + [Epoch 195][50/66] loss=0.940118 avg=0.851866 VRAM=39.0GiB | 39.2% done | ETA(epoch): 308s + [Epoch 195][60/66] loss=0.857839 avg=0.849205 VRAM=38.8GiB | 39.2% done | ETA(epoch): 115s + Train loss: 0.846720 (1269.2s) ETA: 7006min + Val loss: 0.813569 [t_0.0-0.2=1.0742 t_0.2-0.4=0.9754 t_0.4-0.6=0.8270 t_0.6-0.8=0.6792 t_0.8-1.0=0.6290] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 196/499 (39% done) --- + [Epoch 196][10/66] loss=0.767733 avg=0.826820 VRAM=39.0GiB | 39.2% done | ETA(epoch): 1098s + [Epoch 196][20/66] loss=0.729027 avg=0.823357 VRAM=38.8GiB | 39.3% done | ETA(epoch): 893s + [Epoch 196][30/66] loss=0.887192 avg=0.839484 VRAM=39.0GiB | 39.3% done | ETA(epoch): 697s + [Epoch 196][40/66] loss=0.874019 avg=0.840507 VRAM=38.8GiB | 39.3% done | ETA(epoch): 502s + [Epoch 196][50/66] loss=0.794514 avg=0.837665 VRAM=39.0GiB | 39.4% done | ETA(epoch): 309s + [Epoch 196][60/66] loss=0.791755 avg=0.842836 VRAM=38.8GiB | 39.4% done | ETA(epoch): 116s + Train loss: 0.844772 (1273.1s) ETA: 6983min + Val loss: 0.836411 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9952 t_0.4-0.6=0.8560 t_0.6-0.8=0.7129 t_0.8-1.0=0.6163] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 197/499 (39% done) --- + [Epoch 197][10/66] loss=0.832124 avg=0.836035 VRAM=39.0GiB | 39.4% done | ETA(epoch): 1075s + [Epoch 197][20/66] loss=0.818877 avg=0.828581 VRAM=38.8GiB | 39.5% done | ETA(epoch): 883s + [Epoch 197][30/66] loss=0.805738 avg=0.845695 VRAM=39.0GiB | 39.5% done | ETA(epoch): 691s + [Epoch 197][40/66] loss=0.811608 avg=0.837131 VRAM=38.8GiB | 39.5% done | ETA(epoch): 499s + [Epoch 197][50/66] loss=0.843555 avg=0.835338 VRAM=39.0GiB | 39.6% done | ETA(epoch): 307s + [Epoch 197][60/66] loss=0.761149 avg=0.836872 VRAM=38.8GiB | 39.6% done | ETA(epoch): 115s + Train loss: 0.838656 (1270.0s) ETA: 6959min + Val loss: 0.841720 [t_0.0-0.2=1.0650 t_0.2-0.4=0.9792 t_0.4-0.6=0.8417 t_0.6-0.8=0.7191 t_0.8-1.0=0.6366] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 198/499 (40% done) --- + [Epoch 198][10/66] loss=0.929894 avg=0.871571 VRAM=39.0GiB | 39.6% done | ETA(epoch): 1077s + [Epoch 198][20/66] loss=0.863522 avg=0.860781 VRAM=38.8GiB | 39.7% done | ETA(epoch): 884s + [Epoch 198][30/66] loss=1.020064 avg=0.864329 VRAM=39.0GiB | 39.7% done | ETA(epoch): 692s + [Epoch 198][40/66] loss=0.800617 avg=0.849275 VRAM=38.8GiB | 39.7% done | ETA(epoch): 500s + [Epoch 198][50/66] loss=0.938746 avg=0.846788 VRAM=39.0GiB | 39.8% done | ETA(epoch): 307s + [Epoch 198][60/66] loss=0.896625 avg=0.851932 VRAM=38.8GiB | 39.8% done | ETA(epoch): 116s + Train loss: 0.856097 (1270.9s) ETA: 6936min + Val loss: 0.848358 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9896 t_0.4-0.6=0.8554 t_0.6-0.8=0.6993 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 18.5/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 199/499 (40% done) --- + [Epoch 199][10/66] loss=0.927766 avg=0.869454 VRAM=39.0GiB | 39.8% done | ETA(epoch): 1075s + [Epoch 199][20/66] loss=0.813360 avg=0.862446 VRAM=38.8GiB | 39.9% done | ETA(epoch): 883s + [Epoch 199][30/66] loss=0.726782 avg=0.854745 VRAM=39.0GiB | 39.9% done | ETA(epoch): 691s + [Epoch 199][40/66] loss=0.696552 avg=0.849380 VRAM=38.8GiB | 39.9% done | ETA(epoch): 499s + [Epoch 199][50/66] loss=0.793658 avg=0.847420 VRAM=39.0GiB | 40.0% done | ETA(epoch): 307s + [Epoch 199][60/66] loss=0.744273 avg=0.843163 VRAM=38.8GiB | 40.0% done | ETA(epoch): 115s + Train loss: 0.850294 (1267.5s) ETA: 6912min + Val loss: 0.856513 [t_0.0-0.2=1.0754 t_0.2-0.4=0.9835 t_0.4-0.6=0.8137 t_0.6-0.8=0.7346 t_0.8-1.0=0.6177] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 18.6/188.5 GiB (9.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 200/499 (40% done) --- + [MilestoneVis] train_0 step 13200 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 13200 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 13200 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 13200 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 13200 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 13200 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 13200 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 13200 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 13200 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 13200 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 200][10/66] loss=0.763872 avg=0.874404 VRAM=39.0GiB | 40.0% done | ETA(epoch): 6840s + [Epoch 200][20/66] loss=0.813658 avg=0.845685 VRAM=38.8GiB | 40.1% done | ETA(epoch): 3252s + [Epoch 200][30/66] loss=0.811206 avg=0.845194 VRAM=39.0GiB | 40.1% done | ETA(epoch): 1927s + [Epoch 200][40/66] loss=0.780860 avg=0.844734 VRAM=38.8GiB | 40.1% done | ETA(epoch): 1169s + [Epoch 200][50/66] loss=0.729399 avg=0.841708 VRAM=39.0GiB | 40.2% done | ETA(epoch): 637s + [Epoch 200][60/66] loss=0.928981 avg=0.838266 VRAM=38.8GiB | 40.2% done | ETA(epoch): 218s + Train loss: 0.842058 (2297.9s) ETA: 6914min + Val loss: 0.828492 [t_0.0-0.2=1.0667 t_0.2-0.4=1.0141 t_0.4-0.6=0.8611 t_0.6-0.8=0.7176 t_0.8-1.0=0.6317] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 201/499 (40% done) --- + [Epoch 201][10/66] loss=0.872833 avg=0.879523 VRAM=39.0GiB | 40.2% done | ETA(epoch): 1075s + [Epoch 201][20/66] loss=0.937075 avg=0.871403 VRAM=38.8GiB | 40.3% done | ETA(epoch): 884s + [Epoch 201][30/66] loss=0.770461 avg=0.862558 VRAM=39.0GiB | 40.3% done | ETA(epoch): 692s + [Epoch 201][40/66] loss=0.969322 avg=0.865565 VRAM=38.8GiB | 40.3% done | ETA(epoch): 504s + [Epoch 201][50/66] loss=0.752732 avg=0.861232 VRAM=39.0GiB | 40.4% done | ETA(epoch): 309s + [Epoch 201][60/66] loss=0.789427 avg=0.865034 VRAM=38.8GiB | 40.4% done | ETA(epoch): 116s + Train loss: 0.861364 (1274.5s) ETA: 6891min + Val loss: 0.806004 [t_0.0-0.2=1.0741 t_0.2-0.4=0.9766 t_0.4-0.6=0.8490 t_0.6-0.8=0.6950 t_0.8-1.0=0.6386] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 202/499 (40% done) --- + [Epoch 202][10/66] loss=0.826293 avg=0.865783 VRAM=39.0GiB | 40.4% done | ETA(epoch): 1076s + [Epoch 202][20/66] loss=0.825234 avg=0.868832 VRAM=38.8GiB | 40.5% done | ETA(epoch): 884s + [Epoch 202][30/66] loss=0.876319 avg=0.857413 VRAM=39.0GiB | 40.5% done | ETA(epoch): 695s + [Epoch 202][40/66] loss=0.789071 avg=0.858650 VRAM=38.8GiB | 40.5% done | ETA(epoch): 501s + [Epoch 202][50/66] loss=0.703320 avg=0.852938 VRAM=39.0GiB | 40.6% done | ETA(epoch): 308s + [Epoch 202][60/66] loss=0.786588 avg=0.848874 VRAM=38.8GiB | 40.6% done | ETA(epoch): 115s + Train loss: 0.852673 (1269.6s) ETA: 6867min + Val loss: 0.864275 [t_0.0-0.2=1.0583 t_0.2-0.4=1.0135 t_0.4-0.6=0.8320 t_0.6-0.8=0.7081 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 19.3/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 203/499 (41% done) --- + [Epoch 203][10/66] loss=0.850822 avg=0.836315 VRAM=39.0GiB | 40.6% done | ETA(epoch): 1075s + [Epoch 203][20/66] loss=0.902593 avg=0.835235 VRAM=38.8GiB | 40.7% done | ETA(epoch): 884s + [Epoch 203][30/66] loss=0.842593 avg=0.829268 VRAM=39.0GiB | 40.7% done | ETA(epoch): 694s + [Epoch 203][40/66] loss=0.798826 avg=0.832147 VRAM=38.8GiB | 40.7% done | ETA(epoch): 501s + [Epoch 203][50/66] loss=0.868944 avg=0.835020 VRAM=39.0GiB | 40.8% done | ETA(epoch): 308s + [Epoch 203][60/66] loss=1.029198 avg=0.839701 VRAM=38.8GiB | 40.8% done | ETA(epoch): 115s + Train loss: 0.842062 (1269.8s) ETA: 6843min + Val loss: 0.834246 [t_0.0-0.2=1.0627 t_0.2-0.4=0.9940 t_0.4-0.6=0.8523 t_0.6-0.8=0.7237 t_0.8-1.0=0.6280] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 204/499 (41% done) --- + [Epoch 204][10/66] loss=0.764135 avg=0.860167 VRAM=39.0GiB | 40.8% done | ETA(epoch): 1075s + [Epoch 204][20/66] loss=0.942941 avg=0.875091 VRAM=38.8GiB | 40.9% done | ETA(epoch): 887s + [Epoch 204][30/66] loss=0.857944 avg=0.865755 VRAM=39.0GiB | 40.9% done | ETA(epoch): 694s + [Epoch 204][40/66] loss=0.804540 avg=0.856989 VRAM=38.8GiB | 40.9% done | ETA(epoch): 501s + [Epoch 204][50/66] loss=0.978543 avg=0.861222 VRAM=39.0GiB | 41.0% done | ETA(epoch): 308s + [Epoch 204][60/66] loss=0.883260 avg=0.862236 VRAM=38.8GiB | 41.0% done | ETA(epoch): 115s + Train loss: 0.859883 (1269.3s) ETA: 6820min + Val loss: 0.866011 [t_0.0-0.2=1.0655 t_0.2-0.4=0.9885 t_0.4-0.6=0.8613 t_0.6-0.8=0.7059 t_0.8-1.0=0.6325] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 205/499 (41% done) --- + [Epoch 205][10/66] loss=0.901488 avg=0.844576 VRAM=39.0GiB | 41.0% done | ETA(epoch): 1075s + [Epoch 205][20/66] loss=0.887412 avg=0.832564 VRAM=38.8GiB | 41.1% done | ETA(epoch): 883s + [Epoch 205][30/66] loss=0.963020 avg=0.841283 VRAM=39.0GiB | 41.1% done | ETA(epoch): 692s + [Epoch 205][40/66] loss=0.752697 avg=0.835085 VRAM=38.8GiB | 41.1% done | ETA(epoch): 499s + [Epoch 205][50/66] loss=0.898020 avg=0.831128 VRAM=39.0GiB | 41.2% done | ETA(epoch): 307s + [Epoch 205][60/66] loss=0.913966 avg=0.840773 VRAM=38.8GiB | 41.2% done | ETA(epoch): 115s + Train loss: 0.838363 (1267.8s) ETA: 6796min + Val loss: 0.831803 [t_0.0-0.2=1.0857 t_0.2-0.4=0.9853 t_0.4-0.6=0.8366 t_0.6-0.8=0.7030 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 206/499 (41% done) --- + [Epoch 206][10/66] loss=0.946756 avg=0.824317 VRAM=39.0GiB | 41.2% done | ETA(epoch): 1075s + [Epoch 206][20/66] loss=0.930702 avg=0.829178 VRAM=38.8GiB | 41.3% done | ETA(epoch): 883s + [Epoch 206][30/66] loss=0.955828 avg=0.827222 VRAM=39.0GiB | 41.3% done | ETA(epoch): 691s + [Epoch 206][40/66] loss=0.748704 avg=0.828283 VRAM=38.8GiB | 41.3% done | ETA(epoch): 500s + [Epoch 206][50/66] loss=0.758826 avg=0.833700 VRAM=39.0GiB | 41.4% done | ETA(epoch): 307s + [Epoch 206][60/66] loss=0.838958 avg=0.834378 VRAM=38.8GiB | 41.4% done | ETA(epoch): 115s + Train loss: 0.834737 (1268.0s) ETA: 6772min + Val loss: 0.827199 [t_0.0-0.2=1.0632 t_0.2-0.4=1.0514 t_0.4-0.6=0.8317 t_0.6-0.8=0.7151 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 207/499 (41% done) --- + [Epoch 207][10/66] loss=0.845205 avg=0.855908 VRAM=39.0GiB | 41.4% done | ETA(epoch): 1075s + [Epoch 207][20/66] loss=0.834484 avg=0.866113 VRAM=38.8GiB | 41.5% done | ETA(epoch): 884s + [Epoch 207][30/66] loss=0.752339 avg=0.854173 VRAM=39.0GiB | 41.5% done | ETA(epoch): 692s + [Epoch 207][40/66] loss=0.830389 avg=0.851370 VRAM=38.8GiB | 41.5% done | ETA(epoch): 500s + [Epoch 207][50/66] loss=0.855402 avg=0.848254 VRAM=39.0GiB | 41.6% done | ETA(epoch): 308s + [Epoch 207][60/66] loss=1.000875 avg=0.849949 VRAM=38.8GiB | 41.6% done | ETA(epoch): 115s + Train loss: 0.849960 (1269.2s) ETA: 6749min + Val loss: 0.830051 [t_0.0-0.2=1.0833 t_0.2-0.4=1.0128 t_0.4-0.6=0.8319 t_0.6-0.8=0.7196 t_0.8-1.0=0.6228] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 208/499 (42% done) --- + [Epoch 208][10/66] loss=0.833659 avg=0.878907 VRAM=39.0GiB | 41.6% done | ETA(epoch): 1076s + [Epoch 208][20/66] loss=0.851534 avg=0.888213 VRAM=38.8GiB | 41.7% done | ETA(epoch): 885s + [Epoch 208][30/66] loss=0.917952 avg=0.887314 VRAM=39.0GiB | 41.7% done | ETA(epoch): 692s + [Epoch 208][40/66] loss=0.715053 avg=0.863831 VRAM=38.8GiB | 41.7% done | ETA(epoch): 500s + [Epoch 208][50/66] loss=0.824643 avg=0.859908 VRAM=39.0GiB | 41.8% done | ETA(epoch): 308s + [Epoch 208][60/66] loss=0.878554 avg=0.854552 VRAM=38.8GiB | 41.8% done | ETA(epoch): 115s + Train loss: 0.853775 (1269.5s) ETA: 6725min + Val loss: 0.849439 [t_0.0-0.2=1.0550 t_0.2-0.4=1.0111 t_0.4-0.6=0.8234 t_0.6-0.8=0.7268 t_0.8-1.0=0.6287] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 209/499 (42% done) --- + [Epoch 209][10/66] loss=0.928743 avg=0.852308 VRAM=39.0GiB | 41.8% done | ETA(epoch): 1077s + [Epoch 209][20/66] loss=0.980040 avg=0.882071 VRAM=38.8GiB | 41.9% done | ETA(epoch): 884s + [Epoch 209][30/66] loss=0.851163 avg=0.877086 VRAM=39.0GiB | 41.9% done | ETA(epoch): 693s + [Epoch 209][40/66] loss=0.898023 avg=0.862056 VRAM=38.8GiB | 41.9% done | ETA(epoch): 500s + [Epoch 209][50/66] loss=0.936335 avg=0.849386 VRAM=39.0GiB | 42.0% done | ETA(epoch): 308s + [Epoch 209][60/66] loss=0.781914 avg=0.847419 VRAM=38.8GiB | 42.0% done | ETA(epoch): 115s + Train loss: 0.845022 (1269.5s) ETA: 6701min + Val loss: 0.864104 [t_0.0-0.2=1.0728 t_0.2-0.4=1.0172 t_0.4-0.6=0.8461 t_0.6-0.8=0.7087 t_0.8-1.0=0.6285] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 210/499 (42% done) --- + [Epoch 210][10/66] loss=0.861422 avg=0.828721 VRAM=39.0GiB | 42.0% done | ETA(epoch): 1076s + [Epoch 210][20/66] loss=0.898670 avg=0.827783 VRAM=38.8GiB | 42.1% done | ETA(epoch): 885s + [Epoch 210][30/66] loss=0.881720 avg=0.820182 VRAM=39.0GiB | 42.1% done | ETA(epoch): 693s + [Epoch 210][40/66] loss=0.944499 avg=0.821746 VRAM=38.8GiB | 42.1% done | ETA(epoch): 500s + [Epoch 210][50/66] loss=0.850138 avg=0.825672 VRAM=39.0GiB | 42.2% done | ETA(epoch): 308s + [Epoch 210][60/66] loss=0.740058 avg=0.838412 VRAM=38.8GiB | 42.2% done | ETA(epoch): 115s + Train loss: 0.840212 (1270.0s) ETA: 6678min + Val loss: 0.834489 [t_0.0-0.2=1.0745 t_0.2-0.4=0.9974 t_0.4-0.6=0.8627 t_0.6-0.8=0.6974 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 211/499 (42% done) --- + [Epoch 211][10/66] loss=0.920758 avg=0.824424 VRAM=39.0GiB | 42.2% done | ETA(epoch): 1076s + [Epoch 211][20/66] loss=0.846325 avg=0.838382 VRAM=38.8GiB | 42.3% done | ETA(epoch): 889s + [Epoch 211][30/66] loss=0.829990 avg=0.836588 VRAM=39.0GiB | 42.3% done | ETA(epoch): 695s + [Epoch 211][40/66] loss=0.859942 avg=0.845272 VRAM=38.8GiB | 42.3% done | ETA(epoch): 501s + [Epoch 211][50/66] loss=0.873429 avg=0.846288 VRAM=39.0GiB | 42.4% done | ETA(epoch): 308s + [Epoch 211][60/66] loss=0.789103 avg=0.842800 VRAM=38.8GiB | 42.4% done | ETA(epoch): 116s + Train loss: 0.844405 (1271.7s) ETA: 6654min + Val loss: 0.827774 [t_0.0-0.2=1.0541 t_0.2-0.4=1.0034 t_0.4-0.6=0.8019 t_0.6-0.8=0.7012 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 212/499 (42% done) --- + [Epoch 212][10/66] loss=0.755565 avg=0.853821 VRAM=39.0GiB | 42.4% done | ETA(epoch): 1075s + [Epoch 212][20/66] loss=0.780752 avg=0.869102 VRAM=38.8GiB | 42.5% done | ETA(epoch): 884s + [Epoch 212][30/66] loss=0.927556 avg=0.862558 VRAM=39.0GiB | 42.5% done | ETA(epoch): 692s + [Epoch 212][40/66] loss=0.828916 avg=0.858072 VRAM=38.8GiB | 42.5% done | ETA(epoch): 500s + [Epoch 212][50/66] loss=0.726918 avg=0.851973 VRAM=39.0GiB | 42.6% done | ETA(epoch): 308s + [Epoch 212][60/66] loss=0.905507 avg=0.850808 VRAM=38.8GiB | 42.6% done | ETA(epoch): 115s + Train loss: 0.845805 (1268.4s) ETA: 6631min + Val loss: 0.861182 [t_0.0-0.2=1.0829 t_0.2-0.4=0.9798 t_0.4-0.6=0.8410 t_0.6-0.8=0.7068 t_0.8-1.0=0.6285] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 213/499 (43% done) --- + [Epoch 213][10/66] loss=0.832019 avg=0.853027 VRAM=39.0GiB | 42.6% done | ETA(epoch): 1090s + [Epoch 213][20/66] loss=0.804652 avg=0.854645 VRAM=38.8GiB | 42.7% done | ETA(epoch): 890s + [Epoch 213][30/66] loss=0.847724 avg=0.858985 VRAM=39.0GiB | 42.7% done | ETA(epoch): 695s + [Epoch 213][40/66] loss=0.843162 avg=0.858087 VRAM=38.8GiB | 42.7% done | ETA(epoch): 501s + [Epoch 213][50/66] loss=0.922994 avg=0.862382 VRAM=39.0GiB | 42.8% done | ETA(epoch): 308s + [Epoch 213][60/66] loss=0.982048 avg=0.869410 VRAM=38.8GiB | 42.8% done | ETA(epoch): 116s + Train loss: 0.866669 (1270.5s) ETA: 6607min + Val loss: 0.866638 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9895 t_0.4-0.6=0.8543 t_0.6-0.8=0.7072 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 214/499 (43% done) --- + [Epoch 214][10/66] loss=0.716699 avg=0.825609 VRAM=39.0GiB | 42.8% done | ETA(epoch): 1075s + [Epoch 214][20/66] loss=0.831642 avg=0.837011 VRAM=38.8GiB | 42.9% done | ETA(epoch): 883s + [Epoch 214][30/66] loss=0.803909 avg=0.837684 VRAM=39.0GiB | 42.9% done | ETA(epoch): 692s + [Epoch 214][40/66] loss=0.935980 avg=0.835139 VRAM=38.8GiB | 42.9% done | ETA(epoch): 500s + [Epoch 214][50/66] loss=0.822461 avg=0.843532 VRAM=39.0GiB | 43.0% done | ETA(epoch): 307s + [Epoch 214][60/66] loss=0.900537 avg=0.845117 VRAM=38.8GiB | 43.0% done | ETA(epoch): 115s + Train loss: 0.845099 (1268.0s) ETA: 6583min + Val loss: 0.862103 [t_0.0-0.2=1.0642 t_0.2-0.4=0.9940 t_0.4-0.6=0.8527 t_0.6-0.8=0.7388 t_0.8-1.0=0.6304] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 215/499 (43% done) --- + [Epoch 215][10/66] loss=0.839886 avg=0.823318 VRAM=39.0GiB | 43.0% done | ETA(epoch): 1076s + [Epoch 215][20/66] loss=0.966900 avg=0.822344 VRAM=38.8GiB | 43.1% done | ETA(epoch): 884s + [Epoch 215][30/66] loss=0.890748 avg=0.828292 VRAM=39.0GiB | 43.1% done | ETA(epoch): 692s + [Epoch 215][40/66] loss=0.816539 avg=0.836781 VRAM=38.8GiB | 43.1% done | ETA(epoch): 500s + [Epoch 215][50/66] loss=1.019281 avg=0.840050 VRAM=39.0GiB | 43.2% done | ETA(epoch): 308s + [Epoch 215][60/66] loss=0.858796 avg=0.844087 VRAM=38.8GiB | 43.2% done | ETA(epoch): 115s + Train loss: 0.840659 (1270.3s) ETA: 6560min + Val loss: 0.851393 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0217 t_0.4-0.6=0.8050 t_0.6-0.8=0.7001 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 216/499 (43% done) --- + [Epoch 216][10/66] loss=0.752352 avg=0.835511 VRAM=39.0GiB | 43.2% done | ETA(epoch): 1076s + [Epoch 216][20/66] loss=0.702814 avg=0.836170 VRAM=38.8GiB | 43.3% done | ETA(epoch): 884s + [Epoch 216][30/66] loss=0.958836 avg=0.845084 VRAM=39.0GiB | 43.3% done | ETA(epoch): 691s + [Epoch 216][40/66] loss=0.885586 avg=0.846056 VRAM=38.8GiB | 43.3% done | ETA(epoch): 504s + [Epoch 216][50/66] loss=0.917290 avg=0.837443 VRAM=39.0GiB | 43.4% done | ETA(epoch): 310s + [Epoch 216][60/66] loss=0.939816 avg=0.835582 VRAM=38.8GiB | 43.4% done | ETA(epoch): 116s + Train loss: 0.838592 (1274.8s) ETA: 6536min + Val loss: 0.863957 [t_0.0-0.2=1.0637 t_0.2-0.4=1.0266 t_0.4-0.6=0.8330 t_0.6-0.8=0.7260 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 217/499 (43% done) --- + [Epoch 217][10/66] loss=0.966928 avg=0.876418 VRAM=39.0GiB | 43.4% done | ETA(epoch): 1074s + [Epoch 217][20/66] loss=0.991782 avg=0.864737 VRAM=38.8GiB | 43.5% done | ETA(epoch): 883s + [Epoch 217][30/66] loss=0.857168 avg=0.849551 VRAM=39.0GiB | 43.5% done | ETA(epoch): 691s + [Epoch 217][40/66] loss=0.795394 avg=0.844853 VRAM=38.8GiB | 43.5% done | ETA(epoch): 499s + [Epoch 217][50/66] loss=0.930431 avg=0.843300 VRAM=39.0GiB | 43.6% done | ETA(epoch): 307s + [Epoch 217][60/66] loss=0.939510 avg=0.849511 VRAM=38.8GiB | 43.6% done | ETA(epoch): 115s + Train loss: 0.848648 (1267.7s) ETA: 6513min + Val loss: 0.850127 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9767 t_0.4-0.6=0.8120 t_0.6-0.8=0.7366 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 218/499 (44% done) --- + [Epoch 218][10/66] loss=0.718677 avg=0.871786 VRAM=39.0GiB | 43.6% done | ETA(epoch): 1075s + [Epoch 218][20/66] loss=0.870697 avg=0.870225 VRAM=38.8GiB | 43.7% done | ETA(epoch): 883s + [Epoch 218][30/66] loss=0.849146 avg=0.863849 VRAM=39.0GiB | 43.7% done | ETA(epoch): 694s + [Epoch 218][40/66] loss=0.974923 avg=0.868625 VRAM=38.8GiB | 43.7% done | ETA(epoch): 501s + [Epoch 218][50/66] loss=0.992555 avg=0.869475 VRAM=39.0GiB | 43.8% done | ETA(epoch): 308s + [Epoch 218][60/66] loss=0.834050 avg=0.867577 VRAM=38.8GiB | 43.8% done | ETA(epoch): 115s + Train loss: 0.866140 (1269.2s) ETA: 6489min + Val loss: 0.865024 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9842 t_0.4-0.6=0.8181 t_0.6-0.8=0.7068 t_0.8-1.0=0.6288] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 219/499 (44% done) --- + [Epoch 219][10/66] loss=0.892850 avg=0.883983 VRAM=39.0GiB | 43.8% done | ETA(epoch): 1073s + [Epoch 219][20/66] loss=0.728304 avg=0.852068 VRAM=38.8GiB | 43.9% done | ETA(epoch): 891s + [Epoch 219][30/66] loss=0.778612 avg=0.846417 VRAM=39.0GiB | 43.9% done | ETA(epoch): 696s + [Epoch 219][40/66] loss=0.855670 avg=0.852162 VRAM=38.8GiB | 43.9% done | ETA(epoch): 502s + [Epoch 219][50/66] loss=0.817492 avg=0.842162 VRAM=39.0GiB | 44.0% done | ETA(epoch): 308s + [Epoch 219][60/66] loss=0.805690 avg=0.841962 VRAM=38.8GiB | 44.0% done | ETA(epoch): 116s + Train loss: 0.845381 (1270.7s) ETA: 6466min + Val loss: 0.891979 [t_0.0-0.2=1.0528 t_0.2-0.4=1.0247 t_0.4-0.6=0.8548 t_0.6-0.8=0.7138 t_0.8-1.0=0.6193] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 220/499 (44% done) --- + [Epoch 220][10/66] loss=0.830745 avg=0.888843 VRAM=39.0GiB | 44.0% done | ETA(epoch): 1077s + [Epoch 220][20/66] loss=0.828985 avg=0.885751 VRAM=38.8GiB | 44.1% done | ETA(epoch): 884s + [Epoch 220][30/66] loss=0.917688 avg=0.875548 VRAM=39.0GiB | 44.1% done | ETA(epoch): 691s + [Epoch 220][40/66] loss=0.703971 avg=0.865249 VRAM=38.8GiB | 44.1% done | ETA(epoch): 499s + [Epoch 220][50/66] loss=0.935791 avg=0.860162 VRAM=39.0GiB | 44.2% done | ETA(epoch): 307s + [Epoch 220][60/66] loss=0.712527 avg=0.858323 VRAM=38.8GiB | 44.2% done | ETA(epoch): 115s + Train loss: 0.856790 (1266.7s) ETA: 6442min + Val loss: 0.819137 [t_0.0-0.2=1.0858 t_0.2-0.4=0.9736 t_0.4-0.6=0.8337 t_0.6-0.8=0.7088 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 221/499 (44% done) --- + [Epoch 221][10/66] loss=0.727500 avg=0.812764 VRAM=39.0GiB | 44.2% done | ETA(epoch): 1074s + [Epoch 221][20/66] loss=0.905444 avg=0.831634 VRAM=38.8GiB | 44.3% done | ETA(epoch): 883s + [Epoch 221][30/66] loss=0.863493 avg=0.841747 VRAM=39.0GiB | 44.3% done | ETA(epoch): 691s + [Epoch 221][40/66] loss=0.751137 avg=0.846653 VRAM=38.8GiB | 44.3% done | ETA(epoch): 499s + [Epoch 221][50/66] loss=0.886278 avg=0.856675 VRAM=39.0GiB | 44.4% done | ETA(epoch): 307s + [Epoch 221][60/66] loss=0.783621 avg=0.850554 VRAM=38.8GiB | 44.4% done | ETA(epoch): 115s + Train loss: 0.846428 (1266.4s) ETA: 6418min + Val loss: 0.842119 [t_0.0-0.2=1.0781 t_0.2-0.4=0.9850 t_0.4-0.6=0.8314 t_0.6-0.8=0.7049 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 222/499 (44% done) --- + [Epoch 222][10/66] loss=0.947687 avg=0.871549 VRAM=39.0GiB | 44.4% done | ETA(epoch): 1074s + [Epoch 222][20/66] loss=1.026120 avg=0.852107 VRAM=38.8GiB | 44.5% done | ETA(epoch): 882s + [Epoch 222][30/66] loss=0.728742 avg=0.848847 VRAM=39.0GiB | 44.5% done | ETA(epoch): 691s + [Epoch 222][40/66] loss=0.975253 avg=0.855924 VRAM=38.8GiB | 44.5% done | ETA(epoch): 499s + [Epoch 222][50/66] loss=0.855720 avg=0.856162 VRAM=39.0GiB | 44.6% done | ETA(epoch): 307s + [Epoch 222][60/66] loss=0.672259 avg=0.860108 VRAM=38.8GiB | 44.6% done | ETA(epoch): 115s + Train loss: 0.857070 (1266.2s) ETA: 6395min + Val loss: 0.837082 [t_0.0-0.2=1.0703 t_0.2-0.4=0.9475 t_0.4-0.6=0.8547 t_0.6-0.8=0.6989 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 223/499 (45% done) --- + [Epoch 223][10/66] loss=0.906751 avg=0.857135 VRAM=39.0GiB | 44.6% done | ETA(epoch): 1072s + [Epoch 223][20/66] loss=0.754543 avg=0.852901 VRAM=38.8GiB | 44.7% done | ETA(epoch): 880s + [Epoch 223][30/66] loss=0.791490 avg=0.841761 VRAM=39.0GiB | 44.7% done | ETA(epoch): 689s + [Epoch 223][40/66] loss=0.764487 avg=0.840679 VRAM=38.8GiB | 44.7% done | ETA(epoch): 498s + [Epoch 223][50/66] loss=0.851040 avg=0.840327 VRAM=39.0GiB | 44.8% done | ETA(epoch): 306s + [Epoch 223][60/66] loss=0.808566 avg=0.841058 VRAM=38.8GiB | 44.8% done | ETA(epoch): 115s + Train loss: 0.837291 (1263.6s) ETA: 6371min + Val loss: 0.866507 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9793 t_0.4-0.6=0.8547 t_0.6-0.8=0.7111 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 224/499 (45% done) --- + [Epoch 224][10/66] loss=0.724592 avg=0.812747 VRAM=39.0GiB | 44.8% done | ETA(epoch): 1076s + [Epoch 224][20/66] loss=0.858747 avg=0.831078 VRAM=38.8GiB | 44.9% done | ETA(epoch): 883s + [Epoch 224][30/66] loss=0.822930 avg=0.848866 VRAM=39.0GiB | 44.9% done | ETA(epoch): 691s + [Epoch 224][40/66] loss=0.801446 avg=0.846057 VRAM=38.8GiB | 44.9% done | ETA(epoch): 499s + [Epoch 224][50/66] loss=0.973431 avg=0.852863 VRAM=39.0GiB | 45.0% done | ETA(epoch): 307s + [Epoch 224][60/66] loss=0.769720 avg=0.847538 VRAM=38.8GiB | 45.0% done | ETA(epoch): 115s + Train loss: 0.843794 (1265.5s) ETA: 6348min + Val loss: 0.840426 [t_0.0-0.2=1.0853 t_0.2-0.4=1.0032 t_0.4-0.6=0.8395 t_0.6-0.8=0.6909 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 225/499 (45% done) --- + [Epoch 225][10/66] loss=0.882315 avg=0.836923 VRAM=39.0GiB | 45.0% done | ETA(epoch): 1073s + [Epoch 225][20/66] loss=0.681139 avg=0.845834 VRAM=38.8GiB | 45.1% done | ETA(epoch): 881s + [Epoch 225][30/66] loss=0.821406 avg=0.845114 VRAM=39.0GiB | 45.1% done | ETA(epoch): 694s + [Epoch 225][40/66] loss=0.869905 avg=0.846312 VRAM=38.8GiB | 45.1% done | ETA(epoch): 501s + [Epoch 225][50/66] loss=0.803230 avg=0.847889 VRAM=39.0GiB | 45.2% done | ETA(epoch): 308s + [Epoch 225][60/66] loss=0.711802 avg=0.849257 VRAM=38.8GiB | 45.2% done | ETA(epoch): 115s + Train loss: 0.845615 (1268.4s) ETA: 6324min + Val loss: 0.820079 [t_0.0-0.2=1.0773 t_0.2-0.4=0.9980 t_0.4-0.6=0.8210 t_0.6-0.8=0.7019 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 226/499 (45% done) --- + [Epoch 226][10/66] loss=0.835588 avg=0.805741 VRAM=39.0GiB | 45.2% done | ETA(epoch): 1073s + [Epoch 226][20/66] loss=0.886537 avg=0.830797 VRAM=38.8GiB | 45.3% done | ETA(epoch): 881s + [Epoch 226][30/66] loss=0.972434 avg=0.833275 VRAM=39.0GiB | 45.3% done | ETA(epoch): 690s + [Epoch 226][40/66] loss=0.847510 avg=0.834595 VRAM=38.8GiB | 45.3% done | ETA(epoch): 498s + [Epoch 226][50/66] loss=0.890351 avg=0.835533 VRAM=39.0GiB | 45.4% done | ETA(epoch): 307s + [Epoch 226][60/66] loss=0.783370 avg=0.836630 VRAM=38.8GiB | 45.4% done | ETA(epoch): 115s + Train loss: 0.836078 (1265.1s) ETA: 6300min + Val loss: 0.854336 [t_0.0-0.2=1.0734 t_0.2-0.4=0.9915 t_0.4-0.6=0.8549 t_0.6-0.8=0.7294 t_0.8-1.0=0.6189] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 227/499 (45% done) --- + [Epoch 227][10/66] loss=0.855779 avg=0.841296 VRAM=39.0GiB | 45.4% done | ETA(epoch): 1073s + [Epoch 227][20/66] loss=0.964975 avg=0.832435 VRAM=38.8GiB | 45.5% done | ETA(epoch): 882s + [Epoch 227][30/66] loss=0.823901 avg=0.845958 VRAM=39.0GiB | 45.5% done | ETA(epoch): 691s + [Epoch 227][40/66] loss=0.688285 avg=0.840681 VRAM=38.8GiB | 45.5% done | ETA(epoch): 499s + [Epoch 227][50/66] loss=0.916162 avg=0.843788 VRAM=39.0GiB | 45.6% done | ETA(epoch): 307s + [Epoch 227][60/66] loss=0.891784 avg=0.839939 VRAM=38.8GiB | 45.6% done | ETA(epoch): 115s + Train loss: 0.844125 (1266.6s) ETA: 6277min + Val loss: 0.827402 [t_0.0-0.2=1.0590 t_0.2-0.4=0.9955 t_0.4-0.6=0.8777 t_0.6-0.8=0.6984 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 228/499 (46% done) --- + [Epoch 228][10/66] loss=0.826280 avg=0.875351 VRAM=39.0GiB | 45.6% done | ETA(epoch): 1074s + [Epoch 228][20/66] loss=0.923696 avg=0.864230 VRAM=38.8GiB | 45.7% done | ETA(epoch): 882s + [Epoch 228][30/66] loss=0.951424 avg=0.865626 VRAM=39.0GiB | 45.7% done | ETA(epoch): 690s + [Epoch 228][40/66] loss=0.782842 avg=0.868047 VRAM=38.8GiB | 45.7% done | ETA(epoch): 499s + [Epoch 228][50/66] loss=0.844948 avg=0.861688 VRAM=39.0GiB | 45.8% done | ETA(epoch): 307s + [Epoch 228][60/66] loss=0.815164 avg=0.857050 VRAM=38.8GiB | 45.8% done | ETA(epoch): 115s + Train loss: 0.851293 (1266.1s) ETA: 6253min + Val loss: 0.862384 [t_0.0-0.2=1.0645 t_0.2-0.4=1.0058 t_0.4-0.6=0.8487 t_0.6-0.8=0.6797 t_0.8-1.0=0.6142] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 229/499 (46% done) --- + [Epoch 229][10/66] loss=0.908736 avg=0.834824 VRAM=39.0GiB | 45.8% done | ETA(epoch): 1074s + [Epoch 229][20/66] loss=0.864295 avg=0.835338 VRAM=38.8GiB | 45.9% done | ETA(epoch): 882s + [Epoch 229][30/66] loss=0.892894 avg=0.839568 VRAM=39.0GiB | 45.9% done | ETA(epoch): 690s + [Epoch 229][40/66] loss=0.825165 avg=0.844057 VRAM=38.8GiB | 45.9% done | ETA(epoch): 499s + [Epoch 229][50/66] loss=0.873480 avg=0.852399 VRAM=39.0GiB | 46.0% done | ETA(epoch): 307s + [Epoch 229][60/66] loss=0.847618 avg=0.848442 VRAM=38.8GiB | 46.0% done | ETA(epoch): 115s + Train loss: 0.848117 (1266.5s) ETA: 6230min + Val loss: 0.864271 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9933 t_0.4-0.6=0.8496 t_0.6-0.8=0.6893 t_0.8-1.0=0.6385] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 230/499 (46% done) --- + [Epoch 230][10/66] loss=0.897903 avg=0.880597 VRAM=39.0GiB | 46.0% done | ETA(epoch): 1076s + [Epoch 230][20/66] loss=0.821108 avg=0.859297 VRAM=38.8GiB | 46.1% done | ETA(epoch): 883s + [Epoch 230][30/66] loss=0.941227 avg=0.851746 VRAM=39.0GiB | 46.1% done | ETA(epoch): 691s + [Epoch 230][40/66] loss=0.884528 avg=0.862404 VRAM=38.8GiB | 46.1% done | ETA(epoch): 499s + [Epoch 230][50/66] loss=0.872263 avg=0.863109 VRAM=39.0GiB | 46.2% done | ETA(epoch): 307s + [Epoch 230][60/66] loss=0.908492 avg=0.867374 VRAM=38.8GiB | 46.2% done | ETA(epoch): 115s + Train loss: 0.864156 (1266.4s) ETA: 6206min + Val loss: 0.832433 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9973 t_0.4-0.6=0.8161 t_0.6-0.8=0.6936 t_0.8-1.0=0.6108] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 231/499 (46% done) --- + [Epoch 231][10/66] loss=0.862840 avg=0.802095 VRAM=39.0GiB | 46.2% done | ETA(epoch): 1074s + [Epoch 231][20/66] loss=0.867809 avg=0.837781 VRAM=38.8GiB | 46.3% done | ETA(epoch): 883s + [Epoch 231][30/66] loss=0.881149 avg=0.842822 VRAM=39.0GiB | 46.3% done | ETA(epoch): 691s + [Epoch 231][40/66] loss=0.804327 avg=0.832795 VRAM=38.8GiB | 46.3% done | ETA(epoch): 499s + [Epoch 231][50/66] loss=0.879875 avg=0.837121 VRAM=39.0GiB | 46.4% done | ETA(epoch): 308s + [Epoch 231][60/66] loss=0.961782 avg=0.841421 VRAM=38.8GiB | 46.4% done | ETA(epoch): 115s + Train loss: 0.837652 (1268.1s) ETA: 6183min + Val loss: 0.848290 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9594 t_0.4-0.6=0.8415 t_0.6-0.8=0.7191 t_0.8-1.0=0.6158] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 232/499 (46% done) --- + [Epoch 232][10/66] loss=0.808109 avg=0.858271 VRAM=39.0GiB | 46.4% done | ETA(epoch): 1074s + [Epoch 232][20/66] loss=0.962815 avg=0.865166 VRAM=38.8GiB | 46.5% done | ETA(epoch): 883s + [Epoch 232][30/66] loss=0.927532 avg=0.858917 VRAM=39.0GiB | 46.5% done | ETA(epoch): 691s + [Epoch 232][40/66] loss=0.981595 avg=0.867625 VRAM=38.8GiB | 46.5% done | ETA(epoch): 499s + [Epoch 232][50/66] loss=0.728498 avg=0.862699 VRAM=39.0GiB | 46.6% done | ETA(epoch): 307s + [Epoch 232][60/66] loss=0.954551 avg=0.863204 VRAM=38.8GiB | 46.6% done | ETA(epoch): 115s + Train loss: 0.860860 (1266.6s) ETA: 6159min + Val loss: 0.817653 [t_0.0-0.2=1.0799 t_0.2-0.4=0.9738 t_0.4-0.6=0.8075 t_0.6-0.8=0.6873 t_0.8-1.0=0.6318] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 233/499 (47% done) --- + [Epoch 233][10/66] loss=0.798826 avg=0.844617 VRAM=39.0GiB | 46.6% done | ETA(epoch): 1072s + [Epoch 233][20/66] loss=0.901533 avg=0.810028 VRAM=38.8GiB | 46.7% done | ETA(epoch): 881s + [Epoch 233][30/66] loss=0.852935 avg=0.828363 VRAM=39.0GiB | 46.7% done | ETA(epoch): 690s + [Epoch 233][40/66] loss=0.972663 avg=0.833822 VRAM=38.8GiB | 46.7% done | ETA(epoch): 498s + [Epoch 233][50/66] loss=0.802058 avg=0.839479 VRAM=39.0GiB | 46.8% done | ETA(epoch): 307s + [Epoch 233][60/66] loss=0.748202 avg=0.844252 VRAM=38.8GiB | 46.8% done | ETA(epoch): 115s + Train loss: 0.846940 (1265.3s) ETA: 6136min + Val loss: 0.841515 [t_0.0-0.2=1.0580 t_0.2-0.4=0.9974 t_0.4-0.6=0.8376 t_0.6-0.8=0.7234 t_0.8-1.0=0.6416] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 234/499 (47% done) --- + [Epoch 234][10/66] loss=0.758914 avg=0.828955 VRAM=39.0GiB | 46.8% done | ETA(epoch): 1075s + [Epoch 234][20/66] loss=0.813117 avg=0.861926 VRAM=38.8GiB | 46.9% done | ETA(epoch): 882s + [Epoch 234][30/66] loss=0.713456 avg=0.848294 VRAM=39.0GiB | 46.9% done | ETA(epoch): 691s + [Epoch 234][40/66] loss=0.901606 avg=0.855714 VRAM=38.8GiB | 46.9% done | ETA(epoch): 499s + [Epoch 234][50/66] loss=0.955660 avg=0.854499 VRAM=39.0GiB | 47.0% done | ETA(epoch): 307s + [Epoch 234][60/66] loss=0.866541 avg=0.858002 VRAM=38.8GiB | 47.0% done | ETA(epoch): 115s + Train loss: 0.858819 (1266.1s) ETA: 6112min + Val loss: 0.809256 [t_0.0-0.2=1.0701 t_0.2-0.4=1.0219 t_0.4-0.6=0.8580 t_0.6-0.8=0.6972 t_0.8-1.0=0.6254] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 235/499 (47% done) --- + [Epoch 235][10/66] loss=0.717858 avg=0.748159 VRAM=39.0GiB | 47.0% done | ETA(epoch): 1085s + [Epoch 235][20/66] loss=0.829765 avg=0.797529 VRAM=38.8GiB | 47.1% done | ETA(epoch): 886s + [Epoch 235][30/66] loss=0.825034 avg=0.799400 VRAM=39.0GiB | 47.1% done | ETA(epoch): 692s + [Epoch 235][40/66] loss=0.975812 avg=0.818746 VRAM=38.8GiB | 47.1% done | ETA(epoch): 500s + [Epoch 235][50/66] loss=0.877342 avg=0.828360 VRAM=39.0GiB | 47.2% done | ETA(epoch): 307s + [Epoch 235][60/66] loss=0.900777 avg=0.838865 VRAM=38.8GiB | 47.2% done | ETA(epoch): 115s + Train loss: 0.836985 (1267.4s) ETA: 6089min + Val loss: 0.830581 [t_0.0-0.2=1.0685 t_0.2-0.4=0.9846 t_0.4-0.6=0.8349 t_0.6-0.8=0.7072 t_0.8-1.0=0.6142] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 236/499 (47% done) --- + [Epoch 236][10/66] loss=0.877710 avg=0.828578 VRAM=39.0GiB | 47.2% done | ETA(epoch): 1085s + [Epoch 236][20/66] loss=0.852024 avg=0.832057 VRAM=38.8GiB | 47.3% done | ETA(epoch): 886s + [Epoch 236][30/66] loss=0.834148 avg=0.836629 VRAM=39.0GiB | 47.3% done | ETA(epoch): 693s + [Epoch 236][40/66] loss=0.793286 avg=0.840881 VRAM=38.8GiB | 47.3% done | ETA(epoch): 500s + [Epoch 236][50/66] loss=0.691016 avg=0.836330 VRAM=39.0GiB | 47.4% done | ETA(epoch): 307s + [Epoch 236][60/66] loss=0.836656 avg=0.835857 VRAM=38.8GiB | 47.4% done | ETA(epoch): 115s + Train loss: 0.840364 (1267.9s) ETA: 6065min + Val loss: 0.816834 [t_0.0-0.2=1.0804 t_0.2-0.4=0.9654 t_0.4-0.6=0.8403 t_0.6-0.8=0.7228 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 237/499 (47% done) --- + [Epoch 237][10/66] loss=0.850432 avg=0.863853 VRAM=39.0GiB | 47.4% done | ETA(epoch): 1074s + [Epoch 237][20/66] loss=0.882424 avg=0.856066 VRAM=38.8GiB | 47.5% done | ETA(epoch): 883s + [Epoch 237][30/66] loss=1.021373 avg=0.864609 VRAM=39.0GiB | 47.5% done | ETA(epoch): 691s + [Epoch 237][40/66] loss=0.984356 avg=0.867618 VRAM=38.8GiB | 47.5% done | ETA(epoch): 499s + [Epoch 237][50/66] loss=0.736527 avg=0.865408 VRAM=39.0GiB | 47.6% done | ETA(epoch): 307s + [Epoch 237][60/66] loss=0.708171 avg=0.861100 VRAM=38.8GiB | 47.6% done | ETA(epoch): 115s + Train loss: 0.855982 (1266.5s) ETA: 6042min + Val loss: 0.879908 [t_0.0-0.2=1.0736 t_0.2-0.4=1.0012 t_0.4-0.6=0.8481 t_0.6-0.8=0.6925 t_0.8-1.0=0.6215] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 238/499 (48% done) --- + [Epoch 238][10/66] loss=0.958000 avg=0.864346 VRAM=39.0GiB | 47.6% done | ETA(epoch): 1073s + [Epoch 238][20/66] loss=0.915150 avg=0.859192 VRAM=38.8GiB | 47.7% done | ETA(epoch): 882s + [Epoch 238][30/66] loss=0.869126 avg=0.849527 VRAM=39.0GiB | 47.7% done | ETA(epoch): 691s + [Epoch 238][40/66] loss=0.873866 avg=0.835360 VRAM=38.8GiB | 47.7% done | ETA(epoch): 499s + [Epoch 238][50/66] loss=0.804881 avg=0.841234 VRAM=39.0GiB | 47.8% done | ETA(epoch): 308s + [Epoch 238][60/66] loss=0.880405 avg=0.843396 VRAM=38.8GiB | 47.8% done | ETA(epoch): 115s + Train loss: 0.839742 (1268.2s) ETA: 6018min + Val loss: 0.848546 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9966 t_0.4-0.6=0.8262 t_0.6-0.8=0.7061 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 239/499 (48% done) --- + [Epoch 239][10/66] loss=0.906406 avg=0.862344 VRAM=39.0GiB | 47.8% done | ETA(epoch): 1074s + [Epoch 239][20/66] loss=0.777465 avg=0.873967 VRAM=38.8GiB | 47.9% done | ETA(epoch): 882s + [Epoch 239][30/66] loss=0.815287 avg=0.868556 VRAM=39.0GiB | 47.9% done | ETA(epoch): 691s + [Epoch 239][40/66] loss=0.890219 avg=0.872409 VRAM=38.8GiB | 47.9% done | ETA(epoch): 501s + [Epoch 239][50/66] loss=0.858380 avg=0.869222 VRAM=39.0GiB | 48.0% done | ETA(epoch): 308s + [Epoch 239][60/66] loss=0.833630 avg=0.867372 VRAM=38.8GiB | 48.0% done | ETA(epoch): 115s + Train loss: 0.869391 (1269.8s) ETA: 5995min + Val loss: 0.839895 [t_0.0-0.2=1.0808 t_0.2-0.4=1.0011 t_0.4-0.6=0.8311 t_0.6-0.8=0.7019 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 240/499 (48% done) --- + [Epoch 240][10/66] loss=0.916483 avg=0.857512 VRAM=39.0GiB | 48.0% done | ETA(epoch): 1074s + [Epoch 240][20/66] loss=0.832368 avg=0.859069 VRAM=38.8GiB | 48.1% done | ETA(epoch): 882s + [Epoch 240][30/66] loss=0.779351 avg=0.848427 VRAM=39.0GiB | 48.1% done | ETA(epoch): 690s + [Epoch 240][40/66] loss=0.805296 avg=0.858208 VRAM=38.8GiB | 48.1% done | ETA(epoch): 500s + [Epoch 240][50/66] loss=0.903094 avg=0.865442 VRAM=39.0GiB | 48.2% done | ETA(epoch): 308s + [Epoch 240][60/66] loss=0.886189 avg=0.859459 VRAM=38.8GiB | 48.2% done | ETA(epoch): 115s + Train loss: 0.859474 (1268.6s) ETA: 5972min + Val loss: 0.855245 [t_0.0-0.2=1.0821 t_0.2-0.4=0.9930 t_0.4-0.6=0.8235 t_0.6-0.8=0.7151 t_0.8-1.0=0.6132] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 241/499 (48% done) --- + [Epoch 241][10/66] loss=0.803909 avg=0.832737 VRAM=39.0GiB | 48.2% done | ETA(epoch): 1074s + [Epoch 241][20/66] loss=0.808087 avg=0.832422 VRAM=38.8GiB | 48.3% done | ETA(epoch): 882s + [Epoch 241][30/66] loss=0.859062 avg=0.850566 VRAM=39.0GiB | 48.3% done | ETA(epoch): 693s + [Epoch 241][40/66] loss=0.884797 avg=0.847426 VRAM=38.8GiB | 48.3% done | ETA(epoch): 500s + [Epoch 241][50/66] loss=0.973229 avg=0.849745 VRAM=39.0GiB | 48.4% done | ETA(epoch): 308s + [Epoch 241][60/66] loss=0.867867 avg=0.848053 VRAM=38.8GiB | 48.4% done | ETA(epoch): 115s + Train loss: 0.848945 (1268.3s) ETA: 5948min + Val loss: 0.850065 [t_0.0-0.2=1.0729 t_0.2-0.4=0.9804 t_0.4-0.6=0.8543 t_0.6-0.8=0.6937 t_0.8-1.0=0.6502] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 242/499 (48% done) --- + [Epoch 242][10/66] loss=0.696807 avg=0.813941 VRAM=39.0GiB | 48.4% done | ETA(epoch): 1074s + [Epoch 242][20/66] loss=0.775589 avg=0.831198 VRAM=38.8GiB | 48.5% done | ETA(epoch): 882s + [Epoch 242][30/66] loss=0.864204 avg=0.826493 VRAM=39.0GiB | 48.5% done | ETA(epoch): 690s + [Epoch 242][40/66] loss=0.805857 avg=0.831370 VRAM=38.8GiB | 48.5% done | ETA(epoch): 499s + [Epoch 242][50/66] loss=0.931756 avg=0.838646 VRAM=39.0GiB | 48.6% done | ETA(epoch): 307s + [Epoch 242][60/66] loss=0.756818 avg=0.837810 VRAM=38.8GiB | 48.6% done | ETA(epoch): 115s + Train loss: 0.840204 (1265.8s) ETA: 5925min + Val loss: 0.835034 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9478 t_0.4-0.6=0.8953 t_0.6-0.8=0.7175 t_0.8-1.0=0.6252] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 243/499 (49% done) --- + [Epoch 243][10/66] loss=0.844570 avg=0.818464 VRAM=39.0GiB | 48.6% done | ETA(epoch): 1088s + [Epoch 243][20/66] loss=0.869376 avg=0.808414 VRAM=38.8GiB | 48.7% done | ETA(epoch): 887s + [Epoch 243][30/66] loss=0.765161 avg=0.810482 VRAM=39.0GiB | 48.7% done | ETA(epoch): 693s + [Epoch 243][40/66] loss=0.806267 avg=0.815352 VRAM=38.8GiB | 48.7% done | ETA(epoch): 500s + [Epoch 243][50/66] loss=0.915829 avg=0.827766 VRAM=39.0GiB | 48.8% done | ETA(epoch): 308s + [Epoch 243][60/66] loss=0.867781 avg=0.832323 VRAM=38.8GiB | 48.8% done | ETA(epoch): 115s + Train loss: 0.837553 (1268.6s) ETA: 5901min + Val loss: 0.844638 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9661 t_0.4-0.6=0.8526 t_0.6-0.8=0.7065 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 244/499 (49% done) --- + [Epoch 244][10/66] loss=1.017699 avg=0.843554 VRAM=39.0GiB | 48.8% done | ETA(epoch): 1073s + [Epoch 244][20/66] loss=0.966598 avg=0.833864 VRAM=38.8GiB | 48.9% done | ETA(epoch): 882s + [Epoch 244][30/66] loss=1.054569 avg=0.839786 VRAM=39.0GiB | 48.9% done | ETA(epoch): 690s + [Epoch 244][40/66] loss=0.962733 avg=0.845077 VRAM=38.8GiB | 48.9% done | ETA(epoch): 499s + [Epoch 244][50/66] loss=0.937184 avg=0.842076 VRAM=39.0GiB | 49.0% done | ETA(epoch): 307s + [Epoch 244][60/66] loss=0.805984 avg=0.841337 VRAM=38.8GiB | 49.0% done | ETA(epoch): 115s + Train loss: 0.845167 (1265.8s) ETA: 5878min + Val loss: 0.863942 [t_0.0-0.2=1.0530 t_0.2-0.4=0.9861 t_0.4-0.6=0.8450 t_0.6-0.8=0.7163 t_0.8-1.0=0.6336] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 245/499 (49% done) --- + [Epoch 245][10/66] loss=0.839737 avg=0.855149 VRAM=39.0GiB | 49.0% done | ETA(epoch): 1073s + [Epoch 245][20/66] loss=0.897669 avg=0.851400 VRAM=38.8GiB | 49.1% done | ETA(epoch): 883s + [Epoch 245][30/66] loss=0.961602 avg=0.865118 VRAM=39.0GiB | 49.1% done | ETA(epoch): 691s + [Epoch 245][40/66] loss=0.976506 avg=0.857215 VRAM=38.8GiB | 49.1% done | ETA(epoch): 499s + [Epoch 245][50/66] loss=0.967618 avg=0.855579 VRAM=39.0GiB | 49.2% done | ETA(epoch): 307s + [Epoch 245][60/66] loss=0.783367 avg=0.850093 VRAM=38.8GiB | 49.2% done | ETA(epoch): 117s + Train loss: 0.852061 (1284.3s) ETA: 5855min + Val loss: 0.801178 [t_0.0-0.2=1.0606 t_0.2-0.4=1.0217 t_0.4-0.6=0.8350 t_0.6-0.8=0.6834 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0245 (BEST) + Deleted old checkpoint: checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 19.1/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 246/499 (49% done) --- + [Epoch 246][10/66] loss=0.806694 avg=0.845011 VRAM=39.0GiB | 49.2% done | ETA(epoch): 1074s + [Epoch 246][20/66] loss=0.876609 avg=0.833815 VRAM=38.8GiB | 49.3% done | ETA(epoch): 883s + [Epoch 246][30/66] loss=0.857570 avg=0.833330 VRAM=39.0GiB | 49.3% done | ETA(epoch): 691s + [Epoch 246][40/66] loss=0.819935 avg=0.845798 VRAM=38.8GiB | 49.3% done | ETA(epoch): 499s + [Epoch 246][50/66] loss=0.930053 avg=0.849907 VRAM=39.0GiB | 49.4% done | ETA(epoch): 308s + [Epoch 246][60/66] loss=0.870583 avg=0.853060 VRAM=38.8GiB | 49.4% done | ETA(epoch): 115s + Train loss: 0.851482 (1269.4s) ETA: 5831min + Val loss: 0.822305 [t_0.0-0.2=1.0720 t_0.2-0.4=1.0085 t_0.4-0.6=0.8384 t_0.6-0.8=0.7228 t_0.8-1.0=0.6196] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 247/499 (49% done) --- + [Epoch 247][10/66] loss=0.782461 avg=0.888502 VRAM=39.0GiB | 49.4% done | ETA(epoch): 1073s + [Epoch 247][20/66] loss=0.924429 avg=0.891660 VRAM=38.8GiB | 49.5% done | ETA(epoch): 882s + [Epoch 247][30/66] loss=0.661241 avg=0.867161 VRAM=39.0GiB | 49.5% done | ETA(epoch): 691s + [Epoch 247][40/66] loss=0.712082 avg=0.849480 VRAM=38.8GiB | 49.5% done | ETA(epoch): 499s + [Epoch 247][50/66] loss=0.876890 avg=0.848686 VRAM=39.0GiB | 49.6% done | ETA(epoch): 307s + [Epoch 247][60/66] loss=1.022200 avg=0.852682 VRAM=38.8GiB | 49.6% done | ETA(epoch): 115s + Train loss: 0.856760 (1267.0s) ETA: 5808min + Val loss: 0.871725 [t_0.0-0.2=1.0920 t_0.2-0.4=0.9798 t_0.4-0.6=0.8113 t_0.6-0.8=0.7115 t_0.8-1.0=0.6253] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 19.1/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 248/499 (50% done) --- + [Epoch 248][10/66] loss=0.796491 avg=0.841349 VRAM=39.0GiB | 49.6% done | ETA(epoch): 1075s + [Epoch 248][20/66] loss=0.869188 avg=0.861702 VRAM=38.8GiB | 49.7% done | ETA(epoch): 882s + [Epoch 248][30/66] loss=0.848295 avg=0.850519 VRAM=39.0GiB | 49.7% done | ETA(epoch): 696s + [Epoch 248][40/66] loss=0.747250 avg=0.850303 VRAM=38.8GiB | 49.7% done | ETA(epoch): 501s + [Epoch 248][50/66] loss=0.865552 avg=0.849822 VRAM=39.0GiB | 49.8% done | ETA(epoch): 308s + [Epoch 248][60/66] loss=0.928061 avg=0.848777 VRAM=38.8GiB | 49.8% done | ETA(epoch): 116s + Train loss: 0.852157 (1270.2s) ETA: 5785min + Val loss: 0.858591 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9933 t_0.4-0.6=0.8502 t_0.6-0.8=0.7049 t_0.8-1.0=0.6267] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0248 +[MEM @ epoch 248 end] RAM: 19.2/188.5 GiB (10.2%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 249/499 (50% done) --- + [Epoch 249][10/66] loss=0.898284 avg=0.823215 VRAM=39.0GiB | 49.8% done | ETA(epoch): 1077s + [Epoch 249][20/66] loss=0.824297 avg=0.835396 VRAM=38.8GiB | 49.9% done | ETA(epoch): 886s + [Epoch 249][30/66] loss=0.845198 avg=0.850413 VRAM=39.0GiB | 49.9% done | ETA(epoch): 693s + [Epoch 249][40/66] loss=0.782714 avg=0.846988 VRAM=38.8GiB | 49.9% done | ETA(epoch): 500s + [Epoch 249][50/66] loss=0.878662 avg=0.853208 VRAM=39.0GiB | 50.0% done | ETA(epoch): 308s + [Epoch 249][60/66] loss=0.846749 avg=0.851289 VRAM=38.8GiB | 50.0% done | ETA(epoch): 115s + Train loss: 0.853353 (1268.2s) ETA: 5761min + Val loss: 0.848332 [t_0.0-0.2=1.0695 t_0.2-0.4=1.0111 t_0.4-0.6=0.8788 t_0.6-0.8=0.6908 t_0.8-1.0=0.6225] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 19.0/188.5 GiB (10.1%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 250/499 (50% done) --- + [MilestoneVis] train_0 step 16500 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 16500 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 16500 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 16500 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 16500 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 16500 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 16500 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 16500 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 16500 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 16500 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + [Epoch 250][10/66] loss=0.864252 avg=0.818020 VRAM=39.0GiB | 50.0% done | ETA(epoch): 6884s + [Epoch 250][20/66] loss=0.944225 avg=0.837260 VRAM=38.8GiB | 50.1% done | ETA(epoch): 3269s + [Epoch 250][30/66] loss=0.957342 avg=0.847060 VRAM=39.0GiB | 50.1% done | ETA(epoch): 1936s + [Epoch 250][40/66] loss=0.834694 avg=0.842924 VRAM=38.8GiB | 50.1% done | ETA(epoch): 1173s + [Epoch 250][50/66] loss=0.850240 avg=0.839924 VRAM=39.0GiB | 50.2% done | ETA(epoch): 639s + [Epoch 250][60/66] loss=0.919451 avg=0.846148 VRAM=38.8GiB | 50.2% done | ETA(epoch): 219s + Train loss: 0.844270 (2303.9s) ETA: 5755min + Val loss: 0.843244 [t_0.0-0.2=1.0783 t_0.2-0.4=0.9465 t_0.4-0.6=0.8421 t_0.6-0.8=0.7225 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 19.7/188.5 GiB (10.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 251/499 (50% done) --- + [Epoch 251][10/66] loss=0.881935 avg=0.805914 VRAM=39.0GiB | 50.2% done | ETA(epoch): 1072s + [Epoch 251][20/66] loss=0.776765 avg=0.836966 VRAM=38.8GiB | 50.3% done | ETA(epoch): 886s + [Epoch 251][30/66] loss=0.916684 avg=0.836244 VRAM=39.0GiB | 50.3% done | ETA(epoch): 693s + [Epoch 251][40/66] loss=0.749854 avg=0.834086 VRAM=38.8GiB | 50.3% done | ETA(epoch): 500s + [Epoch 251][50/66] loss=0.758061 avg=0.840357 VRAM=39.0GiB | 50.4% done | ETA(epoch): 308s + [Epoch 251][60/66] loss=0.932120 avg=0.843326 VRAM=38.8GiB | 50.4% done | ETA(epoch): 115s + Train loss: 0.851906 (1268.5s) ETA: 5732min + Val loss: 0.871665 [t_0.0-0.2=1.0668 t_0.2-0.4=1.0030 t_0.4-0.6=0.8310 t_0.6-0.8=0.6937 t_0.8-1.0=0.6048] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 19.7/188.5 GiB (10.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 252/499 (50% done) --- + [Epoch 252][10/66] loss=0.864867 avg=0.827380 VRAM=39.0GiB | 50.4% done | ETA(epoch): 1075s + [Epoch 252][20/66] loss=0.746982 avg=0.851564 VRAM=38.8GiB | 50.5% done | ETA(epoch): 883s + [Epoch 252][30/66] loss=0.813190 avg=0.853979 VRAM=39.0GiB | 50.5% done | ETA(epoch): 691s + [Epoch 252][40/66] loss=0.897701 avg=0.854424 VRAM=38.8GiB | 50.5% done | ETA(epoch): 499s + [Epoch 252][50/66] loss=0.867198 avg=0.853151 VRAM=39.0GiB | 50.6% done | ETA(epoch): 307s + [Epoch 252][60/66] loss=0.883770 avg=0.853112 VRAM=38.8GiB | 50.6% done | ETA(epoch): 115s + Train loss: 0.850166 (1265.8s) ETA: 5708min + Val loss: 0.846224 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9629 t_0.4-0.6=0.8276 t_0.6-0.8=0.7086 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 253/499 (51% done) --- + [Epoch 253][10/66] loss=0.957408 avg=0.868170 VRAM=39.0GiB | 50.6% done | ETA(epoch): 1074s + [Epoch 253][20/66] loss=0.782567 avg=0.856744 VRAM=38.8GiB | 50.7% done | ETA(epoch): 883s + [Epoch 253][30/66] loss=0.817081 avg=0.858932 VRAM=39.0GiB | 50.7% done | ETA(epoch): 691s + [Epoch 253][40/66] loss=0.887505 avg=0.848564 VRAM=38.8GiB | 50.7% done | ETA(epoch): 499s + [Epoch 253][50/66] loss=0.819125 avg=0.847908 VRAM=39.0GiB | 50.8% done | ETA(epoch): 307s + [Epoch 253][60/66] loss=0.742782 avg=0.845567 VRAM=38.8GiB | 50.8% done | ETA(epoch): 115s + Train loss: 0.848333 (1266.2s) ETA: 5685min + Val loss: 0.889452 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9930 t_0.4-0.6=0.8307 t_0.6-0.8=0.7169 t_0.8-1.0=0.6210] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 254/499 (51% done) --- + [Epoch 254][10/66] loss=0.867179 avg=0.856170 VRAM=39.0GiB | 50.8% done | ETA(epoch): 1073s + [Epoch 254][20/66] loss=0.792731 avg=0.838287 VRAM=38.8GiB | 50.9% done | ETA(epoch): 882s + [Epoch 254][30/66] loss=0.845960 avg=0.837158 VRAM=39.0GiB | 50.9% done | ETA(epoch): 690s + [Epoch 254][40/66] loss=0.868967 avg=0.848616 VRAM=38.8GiB | 50.9% done | ETA(epoch): 498s + [Epoch 254][50/66] loss=0.914970 avg=0.854178 VRAM=39.0GiB | 51.0% done | ETA(epoch): 307s + [Epoch 254][60/66] loss=0.902225 avg=0.858247 VRAM=38.8GiB | 51.0% done | ETA(epoch): 115s + Train loss: 0.858285 (1267.2s) ETA: 5661min + Val loss: 0.857459 [t_0.0-0.2=1.0717 t_0.2-0.4=0.9912 t_0.4-0.6=0.8706 t_0.6-0.8=0.7039 t_0.8-1.0=0.6242] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 255/499 (51% done) --- + [Epoch 255][10/66] loss=0.767201 avg=0.829020 VRAM=39.0GiB | 51.0% done | ETA(epoch): 1073s + [Epoch 255][20/66] loss=0.832656 avg=0.839118 VRAM=38.8GiB | 51.1% done | ETA(epoch): 882s + [Epoch 255][30/66] loss=0.748953 avg=0.832825 VRAM=39.0GiB | 51.1% done | ETA(epoch): 690s + [Epoch 255][40/66] loss=1.050344 avg=0.844688 VRAM=38.8GiB | 51.1% done | ETA(epoch): 498s + [Epoch 255][50/66] loss=0.802449 avg=0.845839 VRAM=39.0GiB | 51.2% done | ETA(epoch): 307s + [Epoch 255][60/66] loss=0.864515 avg=0.848519 VRAM=38.8GiB | 51.2% done | ETA(epoch): 115s + Train loss: 0.847786 (1265.2s) ETA: 5638min + Val loss: 0.839462 [t_0.0-0.2=1.0567 t_0.2-0.4=0.9570 t_0.4-0.6=0.8595 t_0.6-0.8=0.7179 t_0.8-1.0=0.6212] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 256/499 (51% done) --- + [Epoch 256][10/66] loss=0.904764 avg=0.864215 VRAM=39.0GiB | 51.2% done | ETA(epoch): 1073s + [Epoch 256][20/66] loss=0.805431 avg=0.851509 VRAM=38.8GiB | 51.3% done | ETA(epoch): 883s + [Epoch 256][30/66] loss=0.697580 avg=0.839344 VRAM=39.0GiB | 51.3% done | ETA(epoch): 691s + [Epoch 256][40/66] loss=0.919018 avg=0.849961 VRAM=38.8GiB | 51.3% done | ETA(epoch): 500s + [Epoch 256][50/66] loss=0.885374 avg=0.847517 VRAM=39.0GiB | 51.4% done | ETA(epoch): 307s + [Epoch 256][60/66] loss=0.868051 avg=0.857840 VRAM=38.8GiB | 51.4% done | ETA(epoch): 115s + Train loss: 0.863475 (1267.8s) ETA: 5614min + Val loss: 0.851587 [t_0.0-0.2=1.0597 t_0.2-0.4=0.9904 t_0.4-0.6=0.8535 t_0.6-0.8=0.7045 t_0.8-1.0=0.6251] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 257/499 (51% done) --- + [Epoch 257][10/66] loss=0.904388 avg=0.850830 VRAM=39.0GiB | 51.4% done | ETA(epoch): 1073s + [Epoch 257][20/66] loss=0.906177 avg=0.845816 VRAM=38.8GiB | 51.5% done | ETA(epoch): 882s + [Epoch 257][30/66] loss=0.846103 avg=0.851198 VRAM=39.0GiB | 51.5% done | ETA(epoch): 696s + [Epoch 257][40/66] loss=0.813856 avg=0.847353 VRAM=38.8GiB | 51.5% done | ETA(epoch): 502s + [Epoch 257][50/66] loss=0.901769 avg=0.848589 VRAM=39.0GiB | 51.6% done | ETA(epoch): 308s + [Epoch 257][60/66] loss=0.900772 avg=0.853285 VRAM=38.8GiB | 51.6% done | ETA(epoch): 116s + Train loss: 0.850686 (1270.6s) ETA: 5591min + Val loss: 0.871870 [t_0.0-0.2=1.0738 t_0.2-0.4=1.0162 t_0.4-0.6=0.8437 t_0.6-0.8=0.7081 t_0.8-1.0=0.6104] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 258/499 (52% done) --- + [Epoch 258][10/66] loss=1.010759 avg=0.883119 VRAM=39.0GiB | 51.6% done | ETA(epoch): 1073s + [Epoch 258][20/66] loss=0.908710 avg=0.869924 VRAM=38.8GiB | 51.7% done | ETA(epoch): 882s + [Epoch 258][30/66] loss=0.893336 avg=0.858830 VRAM=39.0GiB | 51.7% done | ETA(epoch): 690s + [Epoch 258][40/66] loss=0.964494 avg=0.864027 VRAM=38.8GiB | 51.7% done | ETA(epoch): 499s + [Epoch 258][50/66] loss=0.720256 avg=0.859133 VRAM=39.0GiB | 51.8% done | ETA(epoch): 307s + [Epoch 258][60/66] loss=0.758329 avg=0.851615 VRAM=38.8GiB | 51.8% done | ETA(epoch): 115s + Train loss: 0.851165 (1265.6s) ETA: 5567min + Val loss: 0.839032 [t_0.0-0.2=1.0664 t_0.2-0.4=1.0048 t_0.4-0.6=0.8304 t_0.6-0.8=0.6922 t_0.8-1.0=0.6278] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 259/499 (52% done) --- + [Epoch 259][10/66] loss=0.944494 avg=0.793757 VRAM=39.0GiB | 51.8% done | ETA(epoch): 1073s + [Epoch 259][20/66] loss=0.899853 avg=0.835058 VRAM=38.8GiB | 51.9% done | ETA(epoch): 882s + [Epoch 259][30/66] loss=0.702512 avg=0.833891 VRAM=39.0GiB | 51.9% done | ETA(epoch): 690s + [Epoch 259][40/66] loss=0.955138 avg=0.831712 VRAM=38.8GiB | 51.9% done | ETA(epoch): 499s + [Epoch 259][50/66] loss=0.872873 avg=0.838473 VRAM=39.0GiB | 52.0% done | ETA(epoch): 307s + [Epoch 259][60/66] loss=0.866873 avg=0.846108 VRAM=38.8GiB | 52.0% done | ETA(epoch): 115s + Train loss: 0.847026 (1266.0s) ETA: 5544min + Val loss: 0.848149 [t_0.0-0.2=1.0687 t_0.2-0.4=1.0023 t_0.4-0.6=0.8386 t_0.6-0.8=0.6935 t_0.8-1.0=0.6233] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 260/499 (52% done) --- + [Epoch 260][10/66] loss=0.785915 avg=0.831694 VRAM=39.0GiB | 52.0% done | ETA(epoch): 1086s + [Epoch 260][20/66] loss=0.702906 avg=0.833078 VRAM=38.8GiB | 52.1% done | ETA(epoch): 888s + [Epoch 260][30/66] loss=0.774945 avg=0.836998 VRAM=39.0GiB | 52.1% done | ETA(epoch): 693s + [Epoch 260][40/66] loss=0.668392 avg=0.833980 VRAM=38.8GiB | 52.1% done | ETA(epoch): 500s + [Epoch 260][50/66] loss=0.871855 avg=0.848022 VRAM=39.0GiB | 52.2% done | ETA(epoch): 308s + [Epoch 260][60/66] loss=0.732682 avg=0.849530 VRAM=38.8GiB | 52.2% done | ETA(epoch): 115s + Train loss: 0.847457 (1273.4s) ETA: 5520min + Val loss: 0.860671 [t_0.0-0.2=1.0697 t_0.2-0.4=1.0108 t_0.4-0.6=0.8426 t_0.6-0.8=0.7052 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 261/499 (52% done) --- + [Epoch 261][10/66] loss=0.857832 avg=0.855307 VRAM=39.0GiB | 52.2% done | ETA(epoch): 1073s + [Epoch 261][20/66] loss=0.768152 avg=0.843924 VRAM=38.8GiB | 52.3% done | ETA(epoch): 881s + [Epoch 261][30/66] loss=0.864206 avg=0.845457 VRAM=39.0GiB | 52.3% done | ETA(epoch): 691s + [Epoch 261][40/66] loss=0.811137 avg=0.833822 VRAM=38.8GiB | 52.3% done | ETA(epoch): 499s + [Epoch 261][50/66] loss=0.806875 avg=0.841422 VRAM=39.0GiB | 52.4% done | ETA(epoch): 307s + [Epoch 261][60/66] loss=0.942151 avg=0.848235 VRAM=38.8GiB | 52.4% done | ETA(epoch): 115s + Train loss: 0.858499 (1266.3s) ETA: 5497min + Val loss: 0.838052 [t_0.0-0.2=1.0556 t_0.2-0.4=1.0193 t_0.4-0.6=0.8396 t_0.6-0.8=0.7031 t_0.8-1.0=0.6215] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 262/499 (52% done) --- + [Epoch 262][10/66] loss=1.008481 avg=0.848908 VRAM=39.0GiB | 52.4% done | ETA(epoch): 1073s + [Epoch 262][20/66] loss=0.711175 avg=0.841484 VRAM=38.8GiB | 52.5% done | ETA(epoch): 882s + [Epoch 262][30/66] loss=0.898633 avg=0.854068 VRAM=39.0GiB | 52.5% done | ETA(epoch): 691s + [Epoch 262][40/66] loss=0.700176 avg=0.850872 VRAM=38.8GiB | 52.5% done | ETA(epoch): 499s + [Epoch 262][50/66] loss=0.809977 avg=0.849204 VRAM=39.0GiB | 52.6% done | ETA(epoch): 308s + [Epoch 262][60/66] loss=0.912768 avg=0.849099 VRAM=38.8GiB | 52.6% done | ETA(epoch): 115s + Train loss: 0.842904 (1269.1s) ETA: 5473min + Val loss: 0.858293 [t_0.0-0.2=1.0647 t_0.2-0.4=0.9785 t_0.4-0.6=0.8311 t_0.6-0.8=0.7064 t_0.8-1.0=0.6042] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 263/499 (53% done) --- + [Epoch 263][10/66] loss=0.996481 avg=0.853974 VRAM=39.0GiB | 52.6% done | ETA(epoch): 1075s + [Epoch 263][20/66] loss=0.859350 avg=0.840405 VRAM=38.8GiB | 52.7% done | ETA(epoch): 882s + [Epoch 263][30/66] loss=0.794792 avg=0.837363 VRAM=39.0GiB | 52.7% done | ETA(epoch): 691s + [Epoch 263][40/66] loss=0.917707 avg=0.836764 VRAM=38.8GiB | 52.7% done | ETA(epoch): 499s + [Epoch 263][50/66] loss=0.761122 avg=0.841056 VRAM=39.0GiB | 52.8% done | ETA(epoch): 307s + [Epoch 263][60/66] loss=0.789156 avg=0.838598 VRAM=38.8GiB | 52.8% done | ETA(epoch): 115s + Train loss: 0.841069 (1266.8s) ETA: 5450min + Val loss: 0.872621 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9897 t_0.4-0.6=0.8694 t_0.6-0.8=0.6954 t_0.8-1.0=0.6206] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 264/499 (53% done) --- + [Epoch 264][10/66] loss=0.882815 avg=0.822704 VRAM=39.0GiB | 52.8% done | ETA(epoch): 1074s + [Epoch 264][20/66] loss=0.839291 avg=0.834633 VRAM=38.8GiB | 52.9% done | ETA(epoch): 883s + [Epoch 264][30/66] loss=0.756092 avg=0.833120 VRAM=39.0GiB | 52.9% done | ETA(epoch): 691s + [Epoch 264][40/66] loss=0.740450 avg=0.828183 VRAM=38.8GiB | 52.9% done | ETA(epoch): 499s + [Epoch 264][50/66] loss=0.813732 avg=0.829158 VRAM=39.0GiB | 53.0% done | ETA(epoch): 307s + [Epoch 264][60/66] loss=0.826727 avg=0.830124 VRAM=38.8GiB | 53.0% done | ETA(epoch): 115s + Train loss: 0.834477 (1266.9s) ETA: 5426min + Val loss: 0.872468 [t_0.0-0.2=1.0580 t_0.2-0.4=1.0139 t_0.4-0.6=0.8755 t_0.6-0.8=0.7179 t_0.8-1.0=0.6120] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 265/499 (53% done) --- + [Epoch 265][10/66] loss=1.035925 avg=0.909774 VRAM=39.0GiB | 53.0% done | ETA(epoch): 1073s + [Epoch 265][20/66] loss=0.915499 avg=0.871237 VRAM=38.8GiB | 53.1% done | ETA(epoch): 881s + [Epoch 265][30/66] loss=0.837631 avg=0.862852 VRAM=39.0GiB | 53.1% done | ETA(epoch): 693s + [Epoch 265][40/66] loss=0.817092 avg=0.861151 VRAM=38.8GiB | 53.1% done | ETA(epoch): 500s + [Epoch 265][50/66] loss=0.689685 avg=0.856298 VRAM=39.0GiB | 53.2% done | ETA(epoch): 307s + [Epoch 265][60/66] loss=0.922758 avg=0.852792 VRAM=38.8GiB | 53.2% done | ETA(epoch): 115s + Train loss: 0.856789 (1267.7s) ETA: 5403min + Val loss: 0.833591 [t_0.0-0.2=1.0914 t_0.2-0.4=0.9832 t_0.4-0.6=0.8611 t_0.6-0.8=0.6837 t_0.8-1.0=0.6257] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 266/499 (53% done) --- + [Epoch 266][10/66] loss=0.909781 avg=0.833391 VRAM=39.0GiB | 53.2% done | ETA(epoch): 1075s + [Epoch 266][20/66] loss=0.894944 avg=0.861812 VRAM=38.8GiB | 53.3% done | ETA(epoch): 882s + [Epoch 266][30/66] loss=0.862628 avg=0.855790 VRAM=39.0GiB | 53.3% done | ETA(epoch): 691s + [Epoch 266][40/66] loss=0.793263 avg=0.849497 VRAM=38.8GiB | 53.3% done | ETA(epoch): 499s + [Epoch 266][50/66] loss=0.764562 avg=0.847703 VRAM=39.0GiB | 53.4% done | ETA(epoch): 307s + [Epoch 266][60/66] loss=0.724138 avg=0.842814 VRAM=38.8GiB | 53.4% done | ETA(epoch): 115s + Train loss: 0.844867 (1266.8s) ETA: 5380min + Val loss: 0.839074 [t_0.0-0.2=1.0658 t_0.2-0.4=0.9834 t_0.4-0.6=0.8505 t_0.6-0.8=0.7272 t_0.8-1.0=0.6340] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 20.1/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 267/499 (53% done) --- + [Epoch 267][10/66] loss=0.743314 avg=0.805428 VRAM=39.0GiB | 53.4% done | ETA(epoch): 1073s + [Epoch 267][20/66] loss=0.867910 avg=0.844728 VRAM=38.8GiB | 53.5% done | ETA(epoch): 882s + [Epoch 267][30/66] loss=0.966607 avg=0.850964 VRAM=39.0GiB | 53.5% done | ETA(epoch): 690s + [Epoch 267][40/66] loss=0.840813 avg=0.861178 VRAM=38.8GiB | 53.5% done | ETA(epoch): 498s + [Epoch 267][50/66] loss=0.767989 avg=0.856455 VRAM=39.0GiB | 53.6% done | ETA(epoch): 307s + [Epoch 267][60/66] loss=0.843731 avg=0.855434 VRAM=38.8GiB | 53.6% done | ETA(epoch): 115s + Train loss: 0.856818 (1265.2s) ETA: 5356min + Val loss: 0.832399 [t_0.0-0.2=1.0644 t_0.2-0.4=0.9831 t_0.4-0.6=0.8274 t_0.6-0.8=0.6908 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 268/499 (54% done) --- + [Epoch 268][10/66] loss=0.723229 avg=0.856929 VRAM=39.0GiB | 53.6% done | ETA(epoch): 1074s + [Epoch 268][20/66] loss=0.975811 avg=0.870483 VRAM=38.8GiB | 53.7% done | ETA(epoch): 882s + [Epoch 268][30/66] loss=0.842184 avg=0.854300 VRAM=39.0GiB | 53.7% done | ETA(epoch): 691s + [Epoch 268][40/66] loss=0.848777 avg=0.848973 VRAM=38.8GiB | 53.7% done | ETA(epoch): 499s + [Epoch 268][50/66] loss=0.827576 avg=0.851354 VRAM=39.0GiB | 53.8% done | ETA(epoch): 307s + [Epoch 268][60/66] loss=0.822335 avg=0.844595 VRAM=38.8GiB | 53.8% done | ETA(epoch): 115s + Train loss: 0.842788 (1266.6s) ETA: 5333min + Val loss: 0.854342 [t_0.0-0.2=1.0488 t_0.2-0.4=1.0301 t_0.4-0.6=0.8435 t_0.6-0.8=0.7071 t_0.8-1.0=0.6060] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 269/499 (54% done) --- + [Epoch 269][10/66] loss=0.828719 avg=0.821689 VRAM=39.0GiB | 53.8% done | ETA(epoch): 1074s + [Epoch 269][20/66] loss=0.939868 avg=0.842050 VRAM=38.8GiB | 53.9% done | ETA(epoch): 883s + [Epoch 269][30/66] loss=0.899383 avg=0.844804 VRAM=39.0GiB | 53.9% done | ETA(epoch): 691s + [Epoch 269][40/66] loss=0.929070 avg=0.851940 VRAM=38.8GiB | 53.9% done | ETA(epoch): 499s + [Epoch 269][50/66] loss=0.860231 avg=0.860103 VRAM=39.0GiB | 54.0% done | ETA(epoch): 307s + [Epoch 269][60/66] loss=0.891953 avg=0.861849 VRAM=38.8GiB | 54.0% done | ETA(epoch): 115s + Train loss: 0.862814 (1268.5s) ETA: 5309min + Val loss: 0.869865 [t_0.0-0.2=1.0637 t_0.2-0.4=0.9739 t_0.4-0.6=0.8728 t_0.6-0.8=0.7123 t_0.8-1.0=0.6219] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 270/499 (54% done) --- + [Epoch 270][10/66] loss=0.918824 avg=0.854038 VRAM=39.0GiB | 54.0% done | ETA(epoch): 1074s + [Epoch 270][20/66] loss=0.854714 avg=0.846645 VRAM=38.8GiB | 54.1% done | ETA(epoch): 883s + [Epoch 270][30/66] loss=0.828381 avg=0.837866 VRAM=39.0GiB | 54.1% done | ETA(epoch): 691s + [Epoch 270][40/66] loss=0.882509 avg=0.848816 VRAM=38.8GiB | 54.1% done | ETA(epoch): 499s + [Epoch 270][50/66] loss=0.875494 avg=0.842376 VRAM=39.0GiB | 54.2% done | ETA(epoch): 307s + [Epoch 270][60/66] loss=0.866468 avg=0.843507 VRAM=38.8GiB | 54.2% done | ETA(epoch): 115s + Train loss: 0.842835 (1267.0s) ETA: 5286min + Val loss: 0.853163 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9714 t_0.4-0.6=0.8513 t_0.6-0.8=0.7200 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 271/499 (54% done) --- + [Epoch 271][10/66] loss=0.854010 avg=0.875873 VRAM=39.0GiB | 54.2% done | ETA(epoch): 1073s + [Epoch 271][20/66] loss=0.871254 avg=0.864033 VRAM=38.8GiB | 54.3% done | ETA(epoch): 883s + [Epoch 271][30/66] loss=0.765571 avg=0.851033 VRAM=39.0GiB | 54.3% done | ETA(epoch): 691s + [Epoch 271][40/66] loss=0.933117 avg=0.858699 VRAM=38.8GiB | 54.3% done | ETA(epoch): 501s + [Epoch 271][50/66] loss=0.865096 avg=0.846939 VRAM=39.0GiB | 54.4% done | ETA(epoch): 308s + [Epoch 271][60/66] loss=0.853515 avg=0.840798 VRAM=38.8GiB | 54.4% done | ETA(epoch): 115s + Train loss: 0.842668 (1269.4s) ETA: 5263min + Val loss: 0.859993 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0320 t_0.4-0.6=0.8488 t_0.6-0.8=0.6870 t_0.8-1.0=0.6242] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 272/499 (54% done) --- + [Epoch 272][10/66] loss=0.841795 avg=0.815418 VRAM=39.0GiB | 54.4% done | ETA(epoch): 1073s + [Epoch 272][20/66] loss=0.728800 avg=0.831161 VRAM=38.8GiB | 54.5% done | ETA(epoch): 882s + [Epoch 272][30/66] loss=0.784847 avg=0.839458 VRAM=39.0GiB | 54.5% done | ETA(epoch): 694s + [Epoch 272][40/66] loss=0.838795 avg=0.829998 VRAM=38.8GiB | 54.5% done | ETA(epoch): 501s + [Epoch 272][50/66] loss=0.883726 avg=0.835426 VRAM=39.0GiB | 54.6% done | ETA(epoch): 308s + [Epoch 272][60/66] loss=0.936443 avg=0.840809 VRAM=38.8GiB | 54.6% done | ETA(epoch): 115s + Train loss: 0.840706 (1269.0s) ETA: 5239min + Val loss: 0.862659 [t_0.0-0.2=1.0658 t_0.2-0.4=1.0447 t_0.4-0.6=0.8513 t_0.6-0.8=0.7041 t_0.8-1.0=0.6107] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 273/499 (55% done) --- + [Epoch 273][10/66] loss=0.846858 avg=0.862414 VRAM=39.0GiB | 54.6% done | ETA(epoch): 1075s + [Epoch 273][20/66] loss=0.869297 avg=0.854032 VRAM=38.8GiB | 54.7% done | ETA(epoch): 883s + [Epoch 273][30/66] loss=0.768364 avg=0.858960 VRAM=39.0GiB | 54.7% done | ETA(epoch): 691s + [Epoch 273][40/66] loss=0.813122 avg=0.862836 VRAM=38.8GiB | 54.7% done | ETA(epoch): 499s + [Epoch 273][50/66] loss=0.801949 avg=0.858593 VRAM=39.0GiB | 54.8% done | ETA(epoch): 307s + [Epoch 273][60/66] loss=0.929245 avg=0.855429 VRAM=38.8GiB | 54.8% done | ETA(epoch): 115s + Train loss: 0.852659 (1266.5s) ETA: 5216min + Val loss: 0.837336 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9895 t_0.4-0.6=0.8068 t_0.6-0.8=0.6983 t_0.8-1.0=0.6136] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 20.4/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 274/499 (55% done) --- + [Epoch 274][10/66] loss=0.808278 avg=0.842693 VRAM=39.0GiB | 54.8% done | ETA(epoch): 1073s + [Epoch 274][20/66] loss=0.938031 avg=0.831645 VRAM=38.8GiB | 54.9% done | ETA(epoch): 886s + [Epoch 274][30/66] loss=0.841562 avg=0.829817 VRAM=39.0GiB | 54.9% done | ETA(epoch): 693s + [Epoch 274][40/66] loss=0.898898 avg=0.830328 VRAM=38.8GiB | 54.9% done | ETA(epoch): 500s + [Epoch 274][50/66] loss=0.666932 avg=0.832433 VRAM=39.0GiB | 55.0% done | ETA(epoch): 308s + [Epoch 274][60/66] loss=0.926055 avg=0.831751 VRAM=38.8GiB | 55.0% done | ETA(epoch): 115s + Train loss: 0.836272 (1269.6s) ETA: 5192min + Val loss: 0.823412 [t_0.0-0.2=1.0337 t_0.2-0.4=0.9918 t_0.4-0.6=0.8205 t_0.6-0.8=0.6963 t_0.8-1.0=0.6339] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 20.3/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 275/499 (55% done) --- + [Epoch 275][10/66] loss=0.889780 avg=0.824392 VRAM=39.0GiB | 55.0% done | ETA(epoch): 1074s + [Epoch 275][20/66] loss=0.902579 avg=0.846559 VRAM=38.8GiB | 55.1% done | ETA(epoch): 883s + [Epoch 275][30/66] loss=0.879671 avg=0.832600 VRAM=39.0GiB | 55.1% done | ETA(epoch): 691s + [Epoch 275][40/66] loss=0.741690 avg=0.832232 VRAM=38.8GiB | 55.1% done | ETA(epoch): 499s + [Epoch 275][50/66] loss=0.791134 avg=0.830704 VRAM=39.0GiB | 55.2% done | ETA(epoch): 307s + [Epoch 275][60/66] loss=0.808895 avg=0.831642 VRAM=38.8GiB | 55.2% done | ETA(epoch): 115s + Train loss: 0.830261 (1266.4s) ETA: 5169min + Val loss: 0.833374 [t_0.0-0.2=1.0657 t_0.2-0.4=0.9761 t_0.4-0.6=0.8485 t_0.6-0.8=0.7243 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 20.3/188.5 GiB (10.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 276/499 (55% done) --- + [Epoch 276][10/66] loss=0.875876 avg=0.863407 VRAM=39.0GiB | 55.2% done | ETA(epoch): 1077s + [Epoch 276][20/66] loss=0.765768 avg=0.854450 VRAM=38.8GiB | 55.3% done | ETA(epoch): 883s + [Epoch 276][30/66] loss=0.853662 avg=0.854969 VRAM=39.0GiB | 55.3% done | ETA(epoch): 691s + [Epoch 276][40/66] loss=0.649002 avg=0.831370 VRAM=38.8GiB | 55.3% done | ETA(epoch): 499s + [Epoch 276][50/66] loss=0.823043 avg=0.842056 VRAM=39.0GiB | 55.4% done | ETA(epoch): 307s + [Epoch 276][60/66] loss=0.741444 avg=0.837406 VRAM=38.8GiB | 55.4% done | ETA(epoch): 115s + Train loss: 0.836495 (1266.9s) ETA: 5146min + Val loss: 0.810512 [t_0.0-0.2=1.0555 t_0.2-0.4=1.0153 t_0.4-0.6=0.8278 t_0.6-0.8=0.6895 t_0.8-1.0=0.6315] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 277/499 (55% done) --- + [Epoch 277][10/66] loss=0.823512 avg=0.819861 VRAM=39.0GiB | 55.4% done | ETA(epoch): 1075s + [Epoch 277][20/66] loss=0.796900 avg=0.820556 VRAM=38.8GiB | 55.5% done | ETA(epoch): 882s + [Epoch 277][30/66] loss=0.889054 avg=0.824867 VRAM=39.0GiB | 55.5% done | ETA(epoch): 691s + [Epoch 277][40/66] loss=0.833393 avg=0.832656 VRAM=38.8GiB | 55.5% done | ETA(epoch): 499s + [Epoch 277][50/66] loss=0.803247 avg=0.832335 VRAM=39.0GiB | 55.6% done | ETA(epoch): 307s + [Epoch 277][60/66] loss=0.725319 avg=0.830982 VRAM=38.8GiB | 55.6% done | ETA(epoch): 115s + Train loss: 0.828368 (1266.8s) ETA: 5122min + Val loss: 0.870170 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9840 t_0.4-0.6=0.8479 t_0.6-0.8=0.7170 t_0.8-1.0=0.6140] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 20.2/188.5 GiB (10.7%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 278/499 (56% done) --- + [Epoch 278][10/66] loss=0.909280 avg=0.891844 VRAM=39.0GiB | 55.6% done | ETA(epoch): 1075s + [Epoch 278][20/66] loss=0.795791 avg=0.879614 VRAM=38.8GiB | 55.7% done | ETA(epoch): 883s + [Epoch 278][30/66] loss=0.973380 avg=0.874197 VRAM=39.0GiB | 55.7% done | ETA(epoch): 691s + [Epoch 278][40/66] loss=0.813103 avg=0.873110 VRAM=38.8GiB | 55.7% done | ETA(epoch): 499s + [Epoch 278][50/66] loss=0.886491 avg=0.871053 VRAM=39.0GiB | 55.8% done | ETA(epoch): 308s + [Epoch 278][60/66] loss=0.857430 avg=0.861311 VRAM=38.8GiB | 55.8% done | ETA(epoch): 115s + Train loss: 0.861615 (1268.9s) ETA: 5099min + Val loss: 0.827376 [t_0.0-0.2=1.0415 t_0.2-0.4=1.0051 t_0.4-0.6=0.8281 t_0.6-0.8=0.7057 t_0.8-1.0=0.6220] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 279/499 (56% done) --- + [Epoch 279][10/66] loss=0.892640 avg=0.844265 VRAM=39.0GiB | 55.8% done | ETA(epoch): 1072s + [Epoch 279][20/66] loss=0.771943 avg=0.844415 VRAM=38.8GiB | 55.9% done | ETA(epoch): 880s + [Epoch 279][30/66] loss=0.841565 avg=0.831147 VRAM=39.0GiB | 55.9% done | ETA(epoch): 689s + [Epoch 279][40/66] loss=0.999242 avg=0.835730 VRAM=38.8GiB | 55.9% done | ETA(epoch): 498s + [Epoch 279][50/66] loss=0.846740 avg=0.845576 VRAM=39.0GiB | 56.0% done | ETA(epoch): 307s + [Epoch 279][60/66] loss=0.854945 avg=0.847269 VRAM=38.8GiB | 56.0% done | ETA(epoch): 115s + Train loss: 0.849833 (1265.7s) ETA: 5076min + Val loss: 0.871992 [t_0.0-0.2=1.0801 t_0.2-0.4=1.0037 t_0.4-0.6=0.8679 t_0.6-0.8=0.7176 t_0.8-1.0=0.6163] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 280/499 (56% done) --- + [Epoch 280][10/66] loss=0.928696 avg=0.869390 VRAM=39.0GiB | 56.0% done | ETA(epoch): 1074s + [Epoch 280][20/66] loss=0.834834 avg=0.855198 VRAM=38.8GiB | 56.1% done | ETA(epoch): 884s + [Epoch 280][30/66] loss=0.926616 avg=0.859290 VRAM=39.0GiB | 56.1% done | ETA(epoch): 698s + [Epoch 280][40/66] loss=0.919106 avg=0.854198 VRAM=38.8GiB | 56.1% done | ETA(epoch): 503s + [Epoch 280][50/66] loss=0.778843 avg=0.856493 VRAM=39.0GiB | 56.2% done | ETA(epoch): 309s + [Epoch 280][60/66] loss=0.848006 avg=0.855780 VRAM=38.8GiB | 56.2% done | ETA(epoch): 116s + Train loss: 0.853262 (1272.9s) ETA: 5052min + Val loss: 0.821158 [t_0.0-0.2=1.0655 t_0.2-0.4=0.9745 t_0.4-0.6=0.8278 t_0.6-0.8=0.7025 t_0.8-1.0=0.6296] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 281/499 (56% done) --- + [Epoch 281][10/66] loss=0.764130 avg=0.825620 VRAM=39.0GiB | 56.2% done | ETA(epoch): 1075s + [Epoch 281][20/66] loss=0.810442 avg=0.818756 VRAM=38.8GiB | 56.3% done | ETA(epoch): 884s + [Epoch 281][30/66] loss=0.931466 avg=0.821848 VRAM=39.0GiB | 56.3% done | ETA(epoch): 692s + [Epoch 281][40/66] loss=0.751986 avg=0.819949 VRAM=38.8GiB | 56.3% done | ETA(epoch): 500s + [Epoch 281][50/66] loss=0.778917 avg=0.831414 VRAM=39.0GiB | 56.4% done | ETA(epoch): 308s + [Epoch 281][60/66] loss=0.741815 avg=0.836938 VRAM=38.8GiB | 56.4% done | ETA(epoch): 115s + Train loss: 0.840117 (1268.5s) ETA: 5029min + Val loss: 0.802216 [t_0.0-0.2=1.0847 t_0.2-0.4=1.0163 t_0.4-0.6=0.8330 t_0.6-0.8=0.7104 t_0.8-1.0=0.6214] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0281 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 282/499 (56% done) --- + [Epoch 282][10/66] loss=0.919471 avg=0.818219 VRAM=39.0GiB | 56.4% done | ETA(epoch): 1076s + [Epoch 282][20/66] loss=0.844150 avg=0.847388 VRAM=38.8GiB | 56.5% done | ETA(epoch): 884s + [Epoch 282][30/66] loss=1.012037 avg=0.851276 VRAM=39.0GiB | 56.5% done | ETA(epoch): 692s + [Epoch 282][40/66] loss=0.917729 avg=0.848238 VRAM=38.8GiB | 56.5% done | ETA(epoch): 500s + [Epoch 282][50/66] loss=0.849491 avg=0.851965 VRAM=39.0GiB | 56.6% done | ETA(epoch): 308s + [Epoch 282][60/66] loss=1.007059 avg=0.854115 VRAM=38.8GiB | 56.6% done | ETA(epoch): 115s + Train loss: 0.852757 (1268.5s) ETA: 5006min + Val loss: 0.843588 [t_0.0-0.2=1.0956 t_0.2-0.4=0.9826 t_0.4-0.6=0.8078 t_0.6-0.8=0.6876 t_0.8-1.0=0.6291] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 283/499 (57% done) --- + [Epoch 283][10/66] loss=0.914820 avg=0.837658 VRAM=39.0GiB | 56.6% done | ETA(epoch): 1075s + [Epoch 283][20/66] loss=0.973629 avg=0.867762 VRAM=38.8GiB | 56.7% done | ETA(epoch): 884s + [Epoch 283][30/66] loss=0.926779 avg=0.879033 VRAM=39.0GiB | 56.7% done | ETA(epoch): 692s + [Epoch 283][40/66] loss=0.823342 avg=0.878603 VRAM=38.8GiB | 56.7% done | ETA(epoch): 500s + [Epoch 283][50/66] loss=0.832189 avg=0.874116 VRAM=39.0GiB | 56.8% done | ETA(epoch): 308s + [Epoch 283][60/66] loss=0.815526 avg=0.868852 VRAM=38.8GiB | 56.8% done | ETA(epoch): 115s + Train loss: 0.869609 (1270.1s) ETA: 4982min + Val loss: 0.878463 [t_0.0-0.2=1.0535 t_0.2-0.4=1.0069 t_0.4-0.6=0.8437 t_0.6-0.8=0.7012 t_0.8-1.0=0.6159] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 284/499 (57% done) --- + [Epoch 284][10/66] loss=0.838119 avg=0.871620 VRAM=39.0GiB | 56.8% done | ETA(epoch): 1079s + [Epoch 284][20/66] loss=0.770596 avg=0.852998 VRAM=38.8GiB | 56.9% done | ETA(epoch): 886s + [Epoch 284][30/66] loss=0.868687 avg=0.847714 VRAM=39.0GiB | 56.9% done | ETA(epoch): 694s + [Epoch 284][40/66] loss=0.803433 avg=0.835363 VRAM=38.8GiB | 56.9% done | ETA(epoch): 501s + [Epoch 284][50/66] loss=0.826626 avg=0.835239 VRAM=39.0GiB | 57.0% done | ETA(epoch): 308s + [Epoch 284][60/66] loss=0.995451 avg=0.845706 VRAM=38.8GiB | 57.0% done | ETA(epoch): 116s + Train loss: 0.845978 (1272.2s) ETA: 4959min + Val loss: 0.851347 [t_0.0-0.2=1.0756 t_0.2-0.4=0.9826 t_0.4-0.6=0.8272 t_0.6-0.8=0.7098 t_0.8-1.0=0.6072] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0284 + Deleted old checkpoint: checkpoint_epoch_0281 +[MEM @ epoch 284 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 285/499 (57% done) --- + [Epoch 285][10/66] loss=0.999366 avg=0.865918 VRAM=39.0GiB | 57.0% done | ETA(epoch): 1078s + [Epoch 285][20/66] loss=0.883895 avg=0.846955 VRAM=38.8GiB | 57.1% done | ETA(epoch): 886s + [Epoch 285][30/66] loss=0.855750 avg=0.856299 VRAM=39.0GiB | 57.1% done | ETA(epoch): 694s + [Epoch 285][40/66] loss=0.780680 avg=0.839858 VRAM=38.8GiB | 57.1% done | ETA(epoch): 502s + [Epoch 285][50/66] loss=0.865396 avg=0.838451 VRAM=39.0GiB | 57.2% done | ETA(epoch): 310s + [Epoch 285][60/66] loss=0.915509 avg=0.836366 VRAM=38.8GiB | 57.2% done | ETA(epoch): 116s + Train loss: 0.835171 (1278.1s) ETA: 4936min + Val loss: 0.820478 [t_0.0-0.2=1.0716 t_0.2-0.4=1.0447 t_0.4-0.6=0.8377 t_0.6-0.8=0.7212 t_0.8-1.0=0.6281] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 286/499 (57% done) --- + [Epoch 286][10/66] loss=0.853624 avg=0.870733 VRAM=39.0GiB | 57.2% done | ETA(epoch): 1079s + [Epoch 286][20/66] loss=0.673933 avg=0.845440 VRAM=38.8GiB | 57.3% done | ETA(epoch): 887s + [Epoch 286][30/66] loss=0.843508 avg=0.839924 VRAM=39.0GiB | 57.3% done | ETA(epoch): 694s + [Epoch 286][40/66] loss=0.808860 avg=0.845679 VRAM=38.8GiB | 57.3% done | ETA(epoch): 503s + [Epoch 286][50/66] loss=0.648824 avg=0.833386 VRAM=39.0GiB | 57.4% done | ETA(epoch): 309s + [Epoch 286][60/66] loss=0.894029 avg=0.837965 VRAM=38.8GiB | 57.4% done | ETA(epoch): 116s + Train loss: 0.832988 (1275.2s) ETA: 4913min + Val loss: 0.834876 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9779 t_0.4-0.6=0.8490 t_0.6-0.8=0.7109 t_0.8-1.0=0.6183] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 287/499 (57% done) --- + [Epoch 287][10/66] loss=1.006057 avg=0.922503 VRAM=39.0GiB | 57.4% done | ETA(epoch): 1081s + [Epoch 287][20/66] loss=0.702346 avg=0.880769 VRAM=38.8GiB | 57.5% done | ETA(epoch): 888s + [Epoch 287][30/66] loss=0.905512 avg=0.876363 VRAM=39.0GiB | 57.5% done | ETA(epoch): 695s + [Epoch 287][40/66] loss=0.743614 avg=0.860513 VRAM=38.8GiB | 57.5% done | ETA(epoch): 502s + [Epoch 287][50/66] loss=0.815882 avg=0.850629 VRAM=39.0GiB | 57.6% done | ETA(epoch): 309s + [Epoch 287][60/66] loss=0.820020 avg=0.855046 VRAM=38.8GiB | 57.6% done | ETA(epoch): 116s + Train loss: 0.855409 (1274.3s) ETA: 4889min + Val loss: 0.854407 [t_0.0-0.2=1.0709 t_0.2-0.4=0.9943 t_0.4-0.6=0.8354 t_0.6-0.8=0.7048 t_0.8-1.0=0.6303] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 288/499 (58% done) --- + [Epoch 288][10/66] loss=0.775863 avg=0.826718 VRAM=39.0GiB | 57.6% done | ETA(epoch): 1080s + [Epoch 288][20/66] loss=0.876997 avg=0.847353 VRAM=38.8GiB | 57.7% done | ETA(epoch): 887s + [Epoch 288][30/66] loss=0.745504 avg=0.843466 VRAM=39.0GiB | 57.7% done | ETA(epoch): 695s + [Epoch 288][40/66] loss=0.856577 avg=0.840113 VRAM=38.8GiB | 57.7% done | ETA(epoch): 502s + [Epoch 288][50/66] loss=0.956075 avg=0.844844 VRAM=39.0GiB | 57.8% done | ETA(epoch): 309s + [Epoch 288][60/66] loss=0.913514 avg=0.843582 VRAM=38.8GiB | 57.8% done | ETA(epoch): 116s + Train loss: 0.840008 (1274.3s) ETA: 4866min + Val loss: 0.875407 [t_0.0-0.2=1.0495 t_0.2-0.4=0.9701 t_0.4-0.6=0.8539 t_0.6-0.8=0.7335 t_0.8-1.0=0.6235] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 21.6/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 289/499 (58% done) --- + [Epoch 289][10/66] loss=0.745452 avg=0.846433 VRAM=39.0GiB | 57.8% done | ETA(epoch): 1082s + [Epoch 289][20/66] loss=0.752103 avg=0.840789 VRAM=38.8GiB | 57.9% done | ETA(epoch): 893s + [Epoch 289][30/66] loss=0.852585 avg=0.838917 VRAM=39.0GiB | 57.9% done | ETA(epoch): 698s + [Epoch 289][40/66] loss=1.002992 avg=0.844434 VRAM=38.8GiB | 57.9% done | ETA(epoch): 503s + [Epoch 289][50/66] loss=0.874276 avg=0.845112 VRAM=39.0GiB | 58.0% done | ETA(epoch): 310s + [Epoch 289][60/66] loss=0.879415 avg=0.846038 VRAM=38.8GiB | 58.0% done | ETA(epoch): 116s + Train loss: 0.848131 (1276.6s) ETA: 4843min + Val loss: 0.896205 [t_0.0-0.2=1.0569 t_0.2-0.4=1.0047 t_0.4-0.6=0.8621 t_0.6-0.8=0.6913 t_0.8-1.0=0.6402] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 21.6/188.5 GiB (11.5%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 290/499 (58% done) --- + [Epoch 290][10/66] loss=0.969454 avg=0.857578 VRAM=39.0GiB | 58.0% done | ETA(epoch): 1079s + [Epoch 290][20/66] loss=0.838725 avg=0.848788 VRAM=38.8GiB | 58.1% done | ETA(epoch): 888s + [Epoch 290][30/66] loss=0.888046 avg=0.839343 VRAM=39.0GiB | 58.1% done | ETA(epoch): 695s + [Epoch 290][40/66] loss=0.785248 avg=0.846368 VRAM=38.8GiB | 58.1% done | ETA(epoch): 502s + [Epoch 290][50/66] loss=0.935677 avg=0.849615 VRAM=39.0GiB | 58.2% done | ETA(epoch): 309s + [Epoch 290][60/66] loss=0.817102 avg=0.845248 VRAM=38.8GiB | 58.2% done | ETA(epoch): 116s + Train loss: 0.843470 (1273.5s) ETA: 4820min + Val loss: 0.859679 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0162 t_0.4-0.6=0.8377 t_0.6-0.8=0.7143 t_0.8-1.0=0.6236] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 291/499 (58% done) --- + [Epoch 291][10/66] loss=0.849001 avg=0.822272 VRAM=39.0GiB | 58.2% done | ETA(epoch): 1080s + [Epoch 291][20/66] loss=0.824470 avg=0.841302 VRAM=38.8GiB | 58.3% done | ETA(epoch): 887s + [Epoch 291][30/66] loss=0.910809 avg=0.855308 VRAM=39.0GiB | 58.3% done | ETA(epoch): 695s + [Epoch 291][40/66] loss=0.832597 avg=0.862625 VRAM=38.8GiB | 58.3% done | ETA(epoch): 502s + [Epoch 291][50/66] loss=0.934042 avg=0.854194 VRAM=39.0GiB | 58.4% done | ETA(epoch): 309s + [Epoch 291][60/66] loss=0.772026 avg=0.847639 VRAM=38.8GiB | 58.4% done | ETA(epoch): 118s + Train loss: 0.847275 (1291.6s) ETA: 4797min + Val loss: 0.842980 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9922 t_0.4-0.6=0.8552 t_0.6-0.8=0.6653 t_0.8-1.0=0.6163] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 292/499 (58% done) --- + [Epoch 292][10/66] loss=0.964820 avg=0.893991 VRAM=39.0GiB | 58.4% done | ETA(epoch): 1082s + [Epoch 292][20/66] loss=0.874234 avg=0.894364 VRAM=38.8GiB | 58.5% done | ETA(epoch): 888s + [Epoch 292][30/66] loss=0.694967 avg=0.871321 VRAM=39.0GiB | 58.5% done | ETA(epoch): 695s + [Epoch 292][40/66] loss=0.799809 avg=0.862348 VRAM=38.8GiB | 58.5% done | ETA(epoch): 502s + [Epoch 292][50/66] loss=0.908594 avg=0.866892 VRAM=39.0GiB | 58.6% done | ETA(epoch): 309s + [Epoch 292][60/66] loss=0.909415 avg=0.861647 VRAM=38.8GiB | 58.6% done | ETA(epoch): 116s + Train loss: 0.860202 (1274.3s) ETA: 4773min + Val loss: 0.861297 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9883 t_0.4-0.6=0.8308 t_0.6-0.8=0.6998 t_0.8-1.0=0.6121] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 293/499 (59% done) --- + [Epoch 293][10/66] loss=0.821360 avg=0.861246 VRAM=39.0GiB | 58.6% done | ETA(epoch): 1082s + [Epoch 293][20/66] loss=0.920642 avg=0.854450 VRAM=38.8GiB | 58.7% done | ETA(epoch): 888s + [Epoch 293][30/66] loss=0.892188 avg=0.860480 VRAM=39.0GiB | 58.7% done | ETA(epoch): 695s + [Epoch 293][40/66] loss=0.842309 avg=0.857628 VRAM=38.8GiB | 58.7% done | ETA(epoch): 502s + [Epoch 293][50/66] loss=0.714278 avg=0.845893 VRAM=39.0GiB | 58.8% done | ETA(epoch): 309s + [Epoch 293][60/66] loss=0.782880 avg=0.844676 VRAM=38.8GiB | 58.8% done | ETA(epoch): 116s + Train loss: 0.847004 (1275.0s) ETA: 4750min + Val loss: 0.847499 [t_0.0-0.2=1.0802 t_0.2-0.4=0.9959 t_0.4-0.6=0.8420 t_0.6-0.8=0.6823 t_0.8-1.0=0.6135] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 294/499 (59% done) --- + [Epoch 294][10/66] loss=0.716704 avg=0.872982 VRAM=39.0GiB | 58.8% done | ETA(epoch): 1084s + [Epoch 294][20/66] loss=0.908018 avg=0.855069 VRAM=38.8GiB | 58.9% done | ETA(epoch): 891s + [Epoch 294][30/66] loss=0.885165 avg=0.856289 VRAM=39.0GiB | 58.9% done | ETA(epoch): 696s + [Epoch 294][40/66] loss=0.879383 avg=0.846029 VRAM=38.8GiB | 58.9% done | ETA(epoch): 506s + [Epoch 294][50/66] loss=0.835947 avg=0.840339 VRAM=39.0GiB | 59.0% done | ETA(epoch): 311s + [Epoch 294][60/66] loss=0.771846 avg=0.842362 VRAM=38.8GiB | 59.0% done | ETA(epoch): 116s + Train loss: 0.845778 (1279.3s) ETA: 4727min + Val loss: 0.849342 [t_0.0-0.2=1.0648 t_0.2-0.4=0.9767 t_0.4-0.6=0.8436 t_0.6-0.8=0.6880 t_0.8-1.0=0.6359] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 21.4/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 295/499 (59% done) --- + [Epoch 295][10/66] loss=0.870425 avg=0.858258 VRAM=39.0GiB | 59.0% done | ETA(epoch): 1079s + [Epoch 295][20/66] loss=0.975546 avg=0.860755 VRAM=38.8GiB | 59.1% done | ETA(epoch): 887s + [Epoch 295][30/66] loss=0.955045 avg=0.862530 VRAM=39.0GiB | 59.1% done | ETA(epoch): 694s + [Epoch 295][40/66] loss=0.946283 avg=0.856903 VRAM=38.8GiB | 59.1% done | ETA(epoch): 501s + [Epoch 295][50/66] loss=0.904608 avg=0.853144 VRAM=39.0GiB | 59.2% done | ETA(epoch): 309s + [Epoch 295][60/66] loss=0.793803 avg=0.853284 VRAM=38.8GiB | 59.2% done | ETA(epoch): 116s + Train loss: 0.853579 (1273.1s) ETA: 4704min + Val loss: 0.821507 [t_0.0-0.2=1.0631 t_0.2-0.4=0.9622 t_0.4-0.6=0.8617 t_0.6-0.8=0.6964 t_0.8-1.0=0.6305] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 21.4/188.5 GiB (11.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 296/499 (59% done) --- + [Epoch 296][10/66] loss=0.891018 avg=0.865981 VRAM=39.0GiB | 59.2% done | ETA(epoch): 1079s + [Epoch 296][20/66] loss=0.887534 avg=0.858667 VRAM=38.8GiB | 59.3% done | ETA(epoch): 886s + [Epoch 296][30/66] loss=0.905805 avg=0.856503 VRAM=39.0GiB | 59.3% done | ETA(epoch): 694s + [Epoch 296][40/66] loss=1.000472 avg=0.860054 VRAM=38.8GiB | 59.3% done | ETA(epoch): 501s + [Epoch 296][50/66] loss=1.016027 avg=0.863864 VRAM=39.0GiB | 59.4% done | ETA(epoch): 308s + [Epoch 296][60/66] loss=0.776970 avg=0.864235 VRAM=38.8GiB | 59.4% done | ETA(epoch): 116s + Train loss: 0.863465 (1273.1s) ETA: 4681min + Val loss: 0.855673 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9489 t_0.4-0.6=0.8217 t_0.6-0.8=0.7188 t_0.8-1.0=0.6292] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 297/499 (59% done) --- + [Epoch 297][10/66] loss=0.927723 avg=0.835005 VRAM=39.0GiB | 59.4% done | ETA(epoch): 1082s + [Epoch 297][20/66] loss=0.752176 avg=0.848138 VRAM=38.8GiB | 59.5% done | ETA(epoch): 889s + [Epoch 297][30/66] loss=0.867263 avg=0.844730 VRAM=39.0GiB | 59.5% done | ETA(epoch): 695s + [Epoch 297][40/66] loss=0.860344 avg=0.847791 VRAM=38.8GiB | 59.5% done | ETA(epoch): 502s + [Epoch 297][50/66] loss=0.856070 avg=0.843595 VRAM=39.0GiB | 59.6% done | ETA(epoch): 309s + [Epoch 297][60/66] loss=0.777858 avg=0.843801 VRAM=38.8GiB | 59.6% done | ETA(epoch): 116s + Train loss: 0.844797 (1273.1s) ETA: 4657min + Val loss: 0.855523 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0032 t_0.4-0.6=0.8187 t_0.6-0.8=0.7029 t_0.8-1.0=0.6074] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 21.5/188.5 GiB (11.4%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 298/499 (60% done) --- + [Epoch 298][10/66] loss=0.865928 avg=0.870704 VRAM=39.0GiB | 59.6% done | ETA(epoch): 1078s + [Epoch 298][20/66] loss=0.913483 avg=0.846398 VRAM=38.8GiB | 59.7% done | ETA(epoch): 886s + [Epoch 298][30/66] loss=0.915743 avg=0.840241 VRAM=39.0GiB | 59.7% done | ETA(epoch): 693s + [Epoch 298][40/66] loss=0.913151 avg=0.849114 VRAM=38.8GiB | 59.7% done | ETA(epoch): 501s + [Epoch 298][50/66] loss=0.807032 avg=0.848064 VRAM=39.0GiB | 59.8% done | ETA(epoch): 308s + [Epoch 298][60/66] loss=0.927697 avg=0.850329 VRAM=38.8GiB | 59.8% done | ETA(epoch): 116s + Train loss: 0.852972 (1271.2s) ETA: 4634min + Val loss: 0.846393 [t_0.0-0.2=1.0572 t_0.2-0.4=0.9750 t_0.4-0.6=0.8787 t_0.6-0.8=0.7288 t_0.8-1.0=0.6212] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 21.3/188.5 GiB (11.3%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 299/499 (60% done) --- + [Epoch 299][10/66] loss=0.893925 avg=0.885499 VRAM=39.0GiB | 59.8% done | ETA(epoch): 1079s + [Epoch 299][20/66] loss=0.747378 avg=0.875647 VRAM=38.8GiB | 59.9% done | ETA(epoch): 887s + [Epoch 299][30/66] loss=0.722678 avg=0.868421 VRAM=39.0GiB | 59.9% done | ETA(epoch): 694s + [Epoch 299][40/66] loss=0.886680 avg=0.849058 VRAM=38.8GiB | 59.9% done | ETA(epoch): 501s + [Epoch 299][50/66] loss=0.891339 avg=0.848694 VRAM=39.0GiB | 60.0% done | ETA(epoch): 308s + [Epoch 299][60/66] loss=0.738250 avg=0.841181 VRAM=38.8GiB | 60.0% done | ETA(epoch): 116s + [MilestoneVis] train_0 step 19799 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 19799 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 19799 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 19799 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 19799 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 19799 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 19799 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 19799 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 19799 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 19799 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_512_H100/milestone_vis/milestone_grid_test_5.png + Train loss: 0.843606 (2306.8s) ETA: 4622min + Val loss: 0.835320 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9564 t_0.4-0.6=0.8641 t_0.6-0.8=0.6977 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_256_dim_512_H100/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 22.2/188.5 GiB (11.8%) | VRAM: 38.4/93.1 GiB (41.3%) + +--- Epoch 300/499 (60% done) --- diff --git a/training log/trainlog(lora256dim768) b/training log/trainlog(lora256dim768) new file mode 100644 index 0000000000000000000000000000000000000000..8e7be4d16c8be3f56367108f4265bd033f30c67c --- /dev/null +++ b/training log/trainlog(lora256dim768) @@ -0,0 +1,3507 @@ + +====================================================================== +Command: train_script.py --name lora_256_dim_768_H200 --batch-size 12 --adapter-lr 1e-3 --num-epochs 300 --lora-lr 1e-4 --adapter-mlp --lora-rank 256 --fusion-dim 768 --seed 42 +====================================================================== +Run dir : output/lora_256_dim_768_H200 +Log file: output/lora_256_dim_768_H200/train.log +GPU: NVIDIA H200 NVL | VRAM: 139.8 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run 1el5497f (0.3s) + m wandb: setting up run 1el5497f (0.3s) + m wandb: setting up run 1el5497f (0.3s) + m wandb: setting up run 1el5497f (0.3s) + m wandb: setting up run 1el5497f (0.3s) + m wandb: setting up run 1el5497f (0.8s) + m wandb: setting up run 1el5497f (0.8s) + m wandb: setting up run 1el5497f (0.8s) + m wandb: setting up run 1el5497f (0.8s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260428_104100-1el5497f +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_256_dim_768_H200-train-20260428-104100 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_256_dim_768_H200 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_256_dim_768_H200/runs/1el5497f + +Run Summary: + name lora_256_dim_768_H200 + fusion MLP (lightweight) + LoRA ON (rank=256) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_256_dim_768_H200 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 768 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 300 + batch_size 12 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_256_dim_768_H200 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 10.0/188.4 GiB (5.3%) | VRAM: 0.0/139.8 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H200 NVL (139.8 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.1s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 10.4/188.4 GiB (5.5%) | VRAM: 0.2/139.8 GiB (0.1%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (35.5s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 12.1/188.4 GiB (6.4%) | VRAM: 37.9/139.8 GiB (27.1%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 14.6M params +Control: 0.0M params +Total trainable: 14.6M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=256, alpha=256.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 78.64M + +Parameter Statistics: + HDC²A Adapter: total=14.6M trainable=14.6M + ControlNet (frozen): total=4212.2M LoRA trainable=78.64M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 14.6M + LoRA 78.64M = 93.20M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~2400 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0439, std=0.5195 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.431632 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.91 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.024048 + depth_encoder.conv_stem.6.weight: 0.022827 + W_s.weight: 0.016968 + mlp_fusion.3.weight: 0.015747 + W_d.weight: 0.015747 + Test result: PASSED +[MEM @ after test] RAM: 12.8/188.4 GiB (6.8%) | VRAM: 38.1/139.8 GiB (27.3%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 839.618 GFLOPs transformer: 273662.180 GFLOPs total: 274501.798 GFLOPs + per-sample forward FLOPs: 274501.798 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=12 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=12 +[Data] Test: 30 samples, batch_size=12 + +====================================================================== +Starting training: 300 epochs × 33 steps = 9900 total steps + batch_size=12, grad_accum=4, world_size=1, effective_bs=48 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 990, 1980, 2970, 3960, 4950, 5939, 6929, 7919, 8909, 9899] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/299 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/33] loss=0.949043 avg=0.928576 VRAM=39.4GiB | 0.1% done | ETA(epoch): 2339s + [Epoch 0][20/33] loss=0.909203 avg=0.919343 VRAM=39.2GiB | 0.2% done | ETA(epoch): 841s + [Epoch 0][30/33] loss=0.863354 avg=0.916682 VRAM=39.4GiB | 0.3% done | ETA(epoch): 157s + Train loss: 0.914685 (1653.3s) ETA: 8252min + Val loss: 0.963370 [t_0.0-0.2=1.0927 t_0.2-0.4=1.0303 t_0.4-0.6=0.8685 t_0.6-0.8=0.7740 t_0.8-1.0=1.1422] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 16.1/188.4 GiB (8.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 1/299 (0% done) --- + [Epoch 1][10/33] loss=0.837156 avg=0.898043 VRAM=39.4GiB | 0.4% done | ETA(epoch): 636s + [Epoch 1][20/33] loss=0.988313 avg=0.908201 VRAM=39.2GiB | 0.5% done | ETA(epoch): 360s + [Epoch 1][30/33] loss=0.907565 avg=0.901111 VRAM=39.4GiB | 0.6% done | ETA(epoch): 83s + Train loss: 0.901693 (913.2s) ETA: 6543min + Val loss: 0.969508 [t_0.0-0.2=1.0790 t_0.2-0.4=1.0542 t_0.4-0.6=0.8994 t_0.6-0.8=0.7884 t_0.8-1.0=1.0269] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0001 +[MEM @ epoch 1 end] RAM: 16.4/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 2/299 (1% done) --- + [Epoch 2][10/33] loss=0.935422 avg=0.920912 VRAM=39.4GiB | 0.8% done | ETA(epoch): 637s + [Epoch 2][20/33] loss=0.876939 avg=0.905826 VRAM=39.2GiB | 0.9% done | ETA(epoch): 360s + [Epoch 2][30/33] loss=0.844890 avg=0.913422 VRAM=39.4GiB | 1.0% done | ETA(epoch): 83s + Train loss: 0.913651 (913.0s) ETA: 5962min + Val loss: 0.961183 [t_0.0-0.2=1.0730 t_0.2-0.4=1.0507 t_0.4-0.6=0.8853 t_0.6-0.8=0.7561 t_0.8-1.0=1.1038] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0002 (BEST) +[MEM @ epoch 2 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 3/299 (1% done) --- + [Epoch 3][10/33] loss=1.071392 avg=0.918386 VRAM=39.4GiB | 1.1% done | ETA(epoch): 637s + [Epoch 3][20/33] loss=0.883996 avg=0.914710 VRAM=39.2GiB | 1.2% done | ETA(epoch): 360s + [Epoch 3][30/33] loss=0.918942 avg=0.908929 VRAM=39.4GiB | 1.3% done | ETA(epoch): 83s + Train loss: 0.904895 (913.3s) ETA: 5664min + Val loss: 0.935554 [t_0.0-0.2=1.1023 t_0.2-0.4=1.0049 t_0.4-0.6=0.8695 t_0.6-0.8=0.7699 t_0.8-1.0=0.9867] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 4/299 (1% done) --- + [Epoch 4][10/33] loss=0.971687 avg=0.924949 VRAM=39.4GiB | 1.4% done | ETA(epoch): 637s + [Epoch 4][20/33] loss=0.939631 avg=0.920994 VRAM=39.2GiB | 1.5% done | ETA(epoch): 360s + [Epoch 4][30/33] loss=0.903602 avg=0.920160 VRAM=39.4GiB | 1.6% done | ETA(epoch): 83s + Train loss: 0.918849 (914.1s) ETA: 5480min + Val loss: 0.949566 [t_0.0-0.2=1.0931 t_0.2-0.4=1.0221 t_0.4-0.6=0.8929 t_0.6-0.8=0.7692 t_0.8-1.0=0.9674] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 5/299 (2% done) --- + [Epoch 5][10/33] loss=0.852861 avg=0.920292 VRAM=39.4GiB | 1.8% done | ETA(epoch): 637s + [Epoch 5][20/33] loss=0.910584 avg=0.915292 VRAM=39.2GiB | 1.9% done | ETA(epoch): 360s + [Epoch 5][30/33] loss=0.831188 avg=0.910517 VRAM=39.4GiB | 2.0% done | ETA(epoch): 83s + Train loss: 0.913842 (915.1s) ETA: 5353min + Val loss: 0.944781 [t_0.0-0.2=1.0934 t_0.2-0.4=1.0230 t_0.4-0.6=0.8514 t_0.6-0.8=0.7575 t_0.8-1.0=0.9171] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0005 + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 6/299 (2% done) --- + [Epoch 6][10/33] loss=0.965403 avg=0.911536 VRAM=39.4GiB | 2.1% done | ETA(epoch): 638s + [Epoch 6][20/33] loss=0.946772 avg=0.908347 VRAM=39.2GiB | 2.2% done | ETA(epoch): 361s + [Epoch 6][30/33] loss=0.860008 avg=0.914841 VRAM=39.4GiB | 2.3% done | ETA(epoch): 83s + Train loss: 0.910192 (915.4s) ETA: 5257min + Val loss: 0.897700 [t_0.0-0.2=1.1000 t_0.2-0.4=0.9803 t_0.4-0.6=0.8673 t_0.6-0.8=0.7833 t_0.8-1.0=0.8259] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0006 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 7/299 (2% done) --- + [Epoch 7][10/33] loss=0.960509 avg=0.926003 VRAM=39.4GiB | 2.4% done | ETA(epoch): 638s + [Epoch 7][20/33] loss=0.893321 avg=0.924362 VRAM=39.2GiB | 2.5% done | ETA(epoch): 361s + [Epoch 7][30/33] loss=0.838575 avg=0.916624 VRAM=39.4GiB | 2.6% done | ETA(epoch): 83s + Train loss: 0.911570 (915.9s) ETA: 5182min + Val loss: 0.925030 [t_0.0-0.2=1.0720 t_0.2-0.4=1.0149 t_0.4-0.6=0.8663 t_0.6-0.8=0.7608 t_0.8-1.0=0.8949] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 8/299 (3% done) --- + [Epoch 8][10/33] loss=0.894905 avg=0.898709 VRAM=39.4GiB | 2.8% done | ETA(epoch): 638s + [Epoch 8][20/33] loss=0.866613 avg=0.886244 VRAM=39.2GiB | 2.9% done | ETA(epoch): 361s + [Epoch 8][30/33] loss=0.948696 avg=0.890071 VRAM=39.4GiB | 3.0% done | ETA(epoch): 83s + Train loss: 0.896977 (915.7s) ETA: 5120min + Val loss: 0.913588 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0360 t_0.4-0.6=0.8545 t_0.6-0.8=0.7458 t_0.8-1.0=0.8094] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 9/299 (3% done) --- + [Epoch 9][10/33] loss=0.936448 avg=0.885773 VRAM=39.4GiB | 3.1% done | ETA(epoch): 638s + [Epoch 9][20/33] loss=0.842071 avg=0.885415 VRAM=39.2GiB | 3.2% done | ETA(epoch): 361s + [Epoch 9][30/33] loss=0.848486 avg=0.886898 VRAM=39.4GiB | 3.3% done | ETA(epoch): 83s + Train loss: 0.886429 (916.5s) ETA: 5067min + Val loss: 0.908759 [t_0.0-0.2=1.0820 t_0.2-0.4=0.9919 t_0.4-0.6=0.8439 t_0.6-0.8=0.7560 t_0.8-1.0=0.8860] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0009 +[MEM @ epoch 9 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 10/299 (3% done) --- + [Epoch 10][10/33] loss=0.816138 avg=0.882339 VRAM=39.4GiB | 3.4% done | ETA(epoch): 639s + [Epoch 10][20/33] loss=0.857725 avg=0.891234 VRAM=39.2GiB | 3.5% done | ETA(epoch): 361s + [Epoch 10][30/33] loss=0.909989 avg=0.899661 VRAM=39.4GiB | 3.6% done | ETA(epoch): 83s + Train loss: 0.898739 (916.3s) ETA: 5020min + Val loss: 0.919810 [t_0.0-0.2=1.0699 t_0.2-0.4=1.0042 t_0.4-0.6=0.8452 t_0.6-0.8=0.7457 t_0.8-1.0=0.9068] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0010 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 15.8/188.4 GiB (8.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 11/299 (4% done) --- + [Epoch 11][10/33] loss=0.862755 avg=0.899736 VRAM=39.4GiB | 3.8% done | ETA(epoch): 639s + [Epoch 11][20/33] loss=0.903671 avg=0.896382 VRAM=39.2GiB | 3.9% done | ETA(epoch): 361s + [Epoch 11][30/33] loss=0.850491 avg=0.887128 VRAM=39.4GiB | 4.0% done | ETA(epoch): 83s + Train loss: 0.885892 (917.6s) ETA: 4980min + Val loss: 0.906333 [t_0.0-0.2=1.0671 t_0.2-0.4=1.0020 t_0.4-0.6=0.8035 t_0.6-0.8=0.7628 t_0.8-1.0=0.8391] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 16.1/188.4 GiB (8.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 12/299 (4% done) --- + [Epoch 12][10/33] loss=0.922249 avg=0.875893 VRAM=39.4GiB | 4.1% done | ETA(epoch): 639s + [Epoch 12][20/33] loss=0.820496 avg=0.882987 VRAM=39.2GiB | 4.2% done | ETA(epoch): 361s + [Epoch 12][30/33] loss=0.839001 avg=0.881716 VRAM=39.4GiB | 4.3% done | ETA(epoch): 83s + Train loss: 0.884085 (917.3s) ETA: 4943min + Val loss: 0.890537 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9821 t_0.4-0.6=0.8483 t_0.6-0.8=0.7429 t_0.8-1.0=0.8038] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0012 (BEST) + Deleted old checkpoint: checkpoint_epoch_0006 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 13/299 (4% done) --- + [Epoch 13][10/33] loss=0.836459 avg=0.865681 VRAM=39.4GiB | 4.4% done | ETA(epoch): 640s + [Epoch 13][20/33] loss=0.764701 avg=0.866391 VRAM=39.2GiB | 4.5% done | ETA(epoch): 362s + [Epoch 13][30/33] loss=0.854734 avg=0.872224 VRAM=39.4GiB | 4.6% done | ETA(epoch): 83s + Train loss: 0.876175 (918.4s) ETA: 4909min + Val loss: 0.894761 [t_0.0-0.2=1.0595 t_0.2-0.4=0.9629 t_0.4-0.6=0.8551 t_0.6-0.8=0.7761 t_0.8-1.0=0.7828] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 16.1/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 14/299 (5% done) --- + [Epoch 14][10/33] loss=0.816062 avg=0.846726 VRAM=39.4GiB | 4.8% done | ETA(epoch): 640s + [Epoch 14][20/33] loss=0.902778 avg=0.884356 VRAM=39.2GiB | 4.9% done | ETA(epoch): 362s + [Epoch 14][30/33] loss=0.928770 avg=0.892273 VRAM=39.4GiB | 5.0% done | ETA(epoch): 84s + Train loss: 0.894457 (918.6s) ETA: 4878min + Val loss: 0.895177 [t_0.0-0.2=1.0785 t_0.2-0.4=0.9986 t_0.4-0.6=0.8580 t_0.6-0.8=0.7516 t_0.8-1.0=0.7630] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0014 + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 16.1/188.4 GiB (8.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 15/299 (5% done) --- + [Epoch 15][10/33] loss=0.928048 avg=0.888772 VRAM=39.4GiB | 5.1% done | ETA(epoch): 640s + [Epoch 15][20/33] loss=0.866241 avg=0.889474 VRAM=39.2GiB | 5.2% done | ETA(epoch): 362s + [Epoch 15][30/33] loss=0.933542 avg=0.887148 VRAM=39.4GiB | 5.3% done | ETA(epoch): 84s + Train loss: 0.889204 (918.6s) ETA: 4849min + Val loss: 0.919359 [t_0.0-0.2=1.0884 t_0.2-0.4=1.0040 t_0.4-0.6=0.8791 t_0.6-0.8=0.7235 t_0.8-1.0=0.7826] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0015 +[MEM @ epoch 15 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 16/299 (5% done) --- + [Epoch 16][10/33] loss=0.865763 avg=0.919655 VRAM=39.4GiB | 5.4% done | ETA(epoch): 640s + [Epoch 16][20/33] loss=0.826438 avg=0.879703 VRAM=39.2GiB | 5.5% done | ETA(epoch): 361s + [Epoch 16][30/33] loss=0.936100 avg=0.881289 VRAM=39.4GiB | 5.6% done | ETA(epoch): 83s + Train loss: 0.878498 (918.3s) ETA: 4821min + Val loss: 0.889686 [t_0.0-0.2=1.0803 t_0.2-0.4=0.9822 t_0.4-0.6=0.8495 t_0.6-0.8=0.7418 t_0.8-1.0=0.7589] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0016 (BEST) + Deleted old checkpoint: checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 17/299 (6% done) --- + [Epoch 17][10/33] loss=0.950965 avg=0.847707 VRAM=39.4GiB | 5.8% done | ETA(epoch): 641s + [Epoch 17][20/33] loss=0.840153 avg=0.859436 VRAM=39.2GiB | 5.9% done | ETA(epoch): 362s + [Epoch 17][30/33] loss=0.937152 avg=0.878058 VRAM=39.4GiB | 6.0% done | ETA(epoch): 84s + Train loss: 0.875271 (919.2s) ETA: 4794min + Val loss: 0.904056 [t_0.0-0.2=1.0809 t_0.2-0.4=1.0241 t_0.4-0.6=0.8518 t_0.6-0.8=0.7289 t_0.8-1.0=0.7386] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 18/299 (6% done) --- + [Epoch 18][10/33] loss=0.850775 avg=0.863290 VRAM=39.4GiB | 6.1% done | ETA(epoch): 640s + [Epoch 18][20/33] loss=0.902041 avg=0.868095 VRAM=39.2GiB | 6.2% done | ETA(epoch): 362s + [Epoch 18][30/33] loss=0.812289 avg=0.851222 VRAM=39.4GiB | 6.3% done | ETA(epoch): 84s + Train loss: 0.851090 (919.2s) ETA: 4769min + Val loss: 0.911106 [t_0.0-0.2=1.0753 t_0.2-0.4=1.0134 t_0.4-0.6=0.8514 t_0.6-0.8=0.7485 t_0.8-1.0=0.7335] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 19/299 (6% done) --- + [Epoch 19][10/33] loss=0.844506 avg=0.894950 VRAM=39.4GiB | 6.4% done | ETA(epoch): 640s + [Epoch 19][20/33] loss=0.887459 avg=0.898138 VRAM=39.2GiB | 6.5% done | ETA(epoch): 362s + [Epoch 19][30/33] loss=0.879137 avg=0.890731 VRAM=39.4GiB | 6.6% done | ETA(epoch): 84s + Train loss: 0.888624 (919.7s) ETA: 4745min + Val loss: 0.872790 [t_0.0-0.2=1.0649 t_0.2-0.4=1.0153 t_0.4-0.6=0.8620 t_0.6-0.8=0.7250 t_0.8-1.0=0.7310] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0019 (BEST) + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 20/299 (7% done) --- + [Epoch 20][10/33] loss=0.908789 avg=0.873256 VRAM=39.4GiB | 6.8% done | ETA(epoch): 639s + [Epoch 20][20/33] loss=0.807248 avg=0.876110 VRAM=39.2GiB | 6.9% done | ETA(epoch): 361s + [Epoch 20][30/33] loss=0.825924 avg=0.875696 VRAM=39.4GiB | 7.0% done | ETA(epoch): 83s + Train loss: 0.873947 (916.6s) ETA: 4720min + Val loss: 0.900791 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9906 t_0.4-0.6=0.8670 t_0.6-0.8=0.7453 t_0.8-1.0=0.7315] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 21/299 (7% done) --- + [Epoch 21][10/33] loss=0.911907 avg=0.871086 VRAM=39.4GiB | 7.1% done | ETA(epoch): 638s + [Epoch 21][20/33] loss=0.858568 avg=0.880725 VRAM=39.2GiB | 7.2% done | ETA(epoch): 361s + [Epoch 21][30/33] loss=0.909194 avg=0.882964 VRAM=39.4GiB | 7.3% done | ETA(epoch): 83s + Train loss: 0.882884 (915.4s) ETA: 4696min + Val loss: 0.847563 [t_0.0-0.2=1.0543 t_0.2-0.4=1.0191 t_0.4-0.6=0.8449 t_0.6-0.8=0.7347 t_0.8-1.0=0.7318] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0021 (BEST) + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 22/299 (7% done) --- + [Epoch 22][10/33] loss=0.871396 avg=0.879468 VRAM=39.4GiB | 7.4% done | ETA(epoch): 638s + [Epoch 22][20/33] loss=0.938610 avg=0.876946 VRAM=39.2GiB | 7.5% done | ETA(epoch): 361s + [Epoch 22][30/33] loss=0.779804 avg=0.871965 VRAM=39.4GiB | 7.6% done | ETA(epoch): 83s + Train loss: 0.871577 (916.0s) ETA: 4673min + Val loss: 0.870053 [t_0.0-0.2=1.0699 t_0.2-0.4=0.9801 t_0.4-0.6=0.8460 t_0.6-0.8=0.7572 t_0.8-1.0=0.6973] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 16.3/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 23/299 (8% done) --- + [Epoch 23][10/33] loss=0.831942 avg=0.869194 VRAM=39.4GiB | 7.8% done | ETA(epoch): 638s + [Epoch 23][20/33] loss=0.864587 avg=0.873951 VRAM=39.2GiB | 7.9% done | ETA(epoch): 360s + [Epoch 23][30/33] loss=0.931645 avg=0.868502 VRAM=39.4GiB | 8.0% done | ETA(epoch): 83s + Train loss: 0.872785 (915.5s) ETA: 4651min + Val loss: 0.868891 [t_0.0-0.2=1.0772 t_0.2-0.4=1.0058 t_0.4-0.6=0.8343 t_0.6-0.8=0.7502 t_0.8-1.0=0.7073] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 24/299 (8% done) --- + [Epoch 24][10/33] loss=0.812003 avg=0.870975 VRAM=39.4GiB | 8.1% done | ETA(epoch): 639s + [Epoch 24][20/33] loss=1.006588 avg=0.871061 VRAM=39.2GiB | 8.2% done | ETA(epoch): 361s + [Epoch 24][30/33] loss=0.880761 avg=0.873358 VRAM=39.4GiB | 8.3% done | ETA(epoch): 83s + Train loss: 0.871402 (917.8s) ETA: 4629min + Val loss: 0.900022 [t_0.0-0.2=1.0812 t_0.2-0.4=0.9814 t_0.4-0.6=0.8302 t_0.6-0.8=0.7337 t_0.8-1.0=0.7342] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0024 +[MEM @ epoch 24 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 25/299 (8% done) --- + [Epoch 25][10/33] loss=0.833666 avg=0.867681 VRAM=39.4GiB | 8.4% done | ETA(epoch): 640s + [Epoch 25][20/33] loss=0.835794 avg=0.860306 VRAM=39.2GiB | 8.5% done | ETA(epoch): 362s + [Epoch 25][30/33] loss=0.890847 avg=0.868110 VRAM=39.4GiB | 8.6% done | ETA(epoch): 83s + Train loss: 0.868206 (918.2s) ETA: 4608min + Val loss: 0.879279 [t_0.0-0.2=1.0816 t_0.2-0.4=0.9827 t_0.4-0.6=0.8387 t_0.6-0.8=0.7432 t_0.8-1.0=0.7258] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 16.4/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 26/299 (9% done) --- + [Epoch 26][10/33] loss=0.887042 avg=0.885611 VRAM=39.4GiB | 8.8% done | ETA(epoch): 638s + [Epoch 26][20/33] loss=0.908298 avg=0.888302 VRAM=39.2GiB | 8.9% done | ETA(epoch): 361s + [Epoch 26][30/33] loss=0.844685 avg=0.883759 VRAM=39.4GiB | 9.0% done | ETA(epoch): 83s + Train loss: 0.881949 (916.1s) ETA: 4586min + Val loss: 0.893670 [t_0.0-0.2=1.0787 t_0.2-0.4=0.9853 t_0.4-0.6=0.8547 t_0.6-0.8=0.7579 t_0.8-1.0=0.7269] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 16.4/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 27/299 (9% done) --- + [Epoch 27][10/33] loss=0.929704 avg=0.873032 VRAM=39.4GiB | 9.1% done | ETA(epoch): 638s + [Epoch 27][20/33] loss=0.864076 avg=0.871162 VRAM=39.2GiB | 9.2% done | ETA(epoch): 361s + [Epoch 27][30/33] loss=0.948237 avg=0.877018 VRAM=39.4GiB | 9.3% done | ETA(epoch): 83s + Train loss: 0.880096 (915.9s) ETA: 4566min + Val loss: 0.883021 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9879 t_0.4-0.6=0.8683 t_0.6-0.8=0.7524 t_0.8-1.0=0.7150] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 16.5/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 28/299 (9% done) --- + [Epoch 28][10/33] loss=0.857185 avg=0.864048 VRAM=39.4GiB | 9.4% done | ETA(epoch): 638s + [Epoch 28][20/33] loss=1.003394 avg=0.869714 VRAM=39.2GiB | 9.5% done | ETA(epoch): 361s + [Epoch 28][30/33] loss=0.861393 avg=0.875058 VRAM=39.4GiB | 9.6% done | ETA(epoch): 83s + Train loss: 0.874505 (915.9s) ETA: 4545min + Val loss: 0.884466 [t_0.0-0.2=1.1026 t_0.2-0.4=1.0211 t_0.4-0.6=0.8614 t_0.6-0.8=0.7165 t_0.8-1.0=0.7262] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 16.3/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 29/299 (10% done) --- + [Epoch 29][10/33] loss=0.897885 avg=0.887698 VRAM=39.4GiB | 9.8% done | ETA(epoch): 638s + [Epoch 29][20/33] loss=0.862026 avg=0.865679 VRAM=39.2GiB | 9.9% done | ETA(epoch): 361s + [Epoch 29][30/33] loss=0.921971 avg=0.881834 VRAM=39.4GiB | 10.0% done | ETA(epoch): 83s + Train loss: 0.879434 (916.7s) ETA: 4525min + Val loss: 0.887265 [t_0.0-0.2=1.0679 t_0.2-0.4=0.9813 t_0.4-0.6=0.8497 t_0.6-0.8=0.7525 t_0.8-1.0=0.7440] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 16.3/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 30/299 (10% done) --- + [MilestoneVis] train_0 step 990 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 990 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 990 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 990 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 990 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 990 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 990 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 990 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 990 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 990 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 30][10/33] loss=0.888499 avg=0.895041 VRAM=39.4GiB | 10.1% done | ETA(epoch): 2363s + [Epoch 30][20/33] loss=0.755974 avg=0.874320 VRAM=39.2GiB | 10.2% done | ETA(epoch): 848s + [Epoch 30][30/33] loss=0.888498 avg=0.870824 VRAM=39.4GiB | 10.3% done | ETA(epoch): 158s + Train loss: 0.869957 (1666.4s) ETA: 4613min + Val loss: 0.880585 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9650 t_0.4-0.6=0.8926 t_0.6-0.8=0.7336 t_0.8-1.0=0.6972] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 31/299 (10% done) --- + [Epoch 31][10/33] loss=0.949053 avg=0.872184 VRAM=39.4GiB | 10.4% done | ETA(epoch): 638s + [Epoch 31][20/33] loss=0.840640 avg=0.861259 VRAM=39.2GiB | 10.5% done | ETA(epoch): 361s + [Epoch 31][30/33] loss=0.894612 avg=0.873320 VRAM=39.4GiB | 10.6% done | ETA(epoch): 83s + Train loss: 0.869535 (915.5s) ETA: 4589min + Val loss: 0.899680 [t_0.0-0.2=1.0645 t_0.2-0.4=1.0516 t_0.4-0.6=0.8345 t_0.6-0.8=0.7279 t_0.8-1.0=0.6958] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 32/299 (11% done) --- + [Epoch 32][10/33] loss=0.850523 avg=0.889883 VRAM=39.4GiB | 10.8% done | ETA(epoch): 638s + [Epoch 32][20/33] loss=0.838512 avg=0.889061 VRAM=39.2GiB | 10.9% done | ETA(epoch): 361s + [Epoch 32][30/33] loss=0.804806 avg=0.880011 VRAM=39.4GiB | 11.0% done | ETA(epoch): 83s + Train loss: 0.880582 (916.4s) ETA: 4566min + Val loss: 0.879007 [t_0.0-0.2=1.0764 t_0.2-0.4=0.9829 t_0.4-0.6=0.8485 t_0.6-0.8=0.7465 t_0.8-1.0=0.7076] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 33/299 (11% done) --- + [Epoch 33][10/33] loss=0.828276 avg=0.889247 VRAM=39.4GiB | 11.1% done | ETA(epoch): 639s + [Epoch 33][20/33] loss=0.824305 avg=0.884933 VRAM=39.2GiB | 11.2% done | ETA(epoch): 361s + [Epoch 33][30/33] loss=0.926558 avg=0.886280 VRAM=39.4GiB | 11.3% done | ETA(epoch): 83s + Train loss: 0.883717 (916.7s) ETA: 4544min + Val loss: 0.869605 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9824 t_0.4-0.6=0.8497 t_0.6-0.8=0.7390 t_0.8-1.0=0.6871] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 34/299 (11% done) --- + [Epoch 34][10/33] loss=0.853291 avg=0.856393 VRAM=39.4GiB | 11.4% done | ETA(epoch): 639s + [Epoch 34][20/33] loss=0.898732 avg=0.857313 VRAM=39.2GiB | 11.5% done | ETA(epoch): 361s + [Epoch 34][30/33] loss=0.815380 avg=0.863138 VRAM=39.4GiB | 11.6% done | ETA(epoch): 83s + Train loss: 0.863901 (917.0s) ETA: 4521min + Val loss: 0.849417 [t_0.0-0.2=1.0820 t_0.2-0.4=1.0031 t_0.4-0.6=0.8212 t_0.6-0.8=0.7268 t_0.8-1.0=0.6935] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 35/299 (12% done) --- + [Epoch 35][10/33] loss=0.835600 avg=0.836502 VRAM=39.4GiB | 11.8% done | ETA(epoch): 639s + [Epoch 35][20/33] loss=0.814858 avg=0.845754 VRAM=39.2GiB | 11.9% done | ETA(epoch): 361s + [Epoch 35][30/33] loss=0.717591 avg=0.845763 VRAM=39.4GiB | 12.0% done | ETA(epoch): 83s + Train loss: 0.843347 (917.5s) ETA: 4499min + Val loss: 0.841561 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9876 t_0.4-0.6=0.8463 t_0.6-0.8=0.7109 t_0.8-1.0=0.6805] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0035 (BEST) + Deleted old checkpoint: checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 36/299 (12% done) --- + [Epoch 36][10/33] loss=0.836927 avg=0.833545 VRAM=39.4GiB | 12.1% done | ETA(epoch): 639s + [Epoch 36][20/33] loss=0.921668 avg=0.852932 VRAM=39.2GiB | 12.2% done | ETA(epoch): 361s + [Epoch 36][30/33] loss=0.965623 avg=0.866286 VRAM=39.4GiB | 12.3% done | ETA(epoch): 83s + Train loss: 0.866791 (917.3s) ETA: 4478min + Val loss: 0.889437 [t_0.0-0.2=1.0811 t_0.2-0.4=0.9618 t_0.4-0.6=0.8571 t_0.6-0.8=0.7308 t_0.8-1.0=0.6763] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 37/299 (12% done) --- + [Epoch 37][10/33] loss=0.901862 avg=0.853451 VRAM=39.4GiB | 12.4% done | ETA(epoch): 638s + [Epoch 37][20/33] loss=0.861195 avg=0.864094 VRAM=39.2GiB | 12.5% done | ETA(epoch): 361s + [Epoch 37][30/33] loss=0.883211 avg=0.864852 VRAM=39.4GiB | 12.6% done | ETA(epoch): 83s + Train loss: 0.865205 (916.0s) ETA: 4456min + Val loss: 0.865390 [t_0.0-0.2=1.0776 t_0.2-0.4=1.0297 t_0.4-0.6=0.8559 t_0.6-0.8=0.7358 t_0.8-1.0=0.6976] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 38/299 (13% done) --- + [Epoch 38][10/33] loss=0.904229 avg=0.842605 VRAM=39.4GiB | 12.8% done | ETA(epoch): 638s + [Epoch 38][20/33] loss=0.818620 avg=0.856276 VRAM=39.2GiB | 12.9% done | ETA(epoch): 361s + [Epoch 38][30/33] loss=0.863355 avg=0.848457 VRAM=39.4GiB | 13.0% done | ETA(epoch): 83s + Train loss: 0.853216 (916.0s) ETA: 4435min + Val loss: 0.857385 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9987 t_0.4-0.6=0.8504 t_0.6-0.8=0.7376 t_0.8-1.0=0.6785] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0038 +[MEM @ epoch 38 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 39/299 (13% done) --- + [Epoch 39][10/33] loss=0.899443 avg=0.865705 VRAM=39.4GiB | 13.1% done | ETA(epoch): 638s + [Epoch 39][20/33] loss=0.852480 avg=0.867850 VRAM=39.2GiB | 13.2% done | ETA(epoch): 361s + [Epoch 39][30/33] loss=0.848457 avg=0.875424 VRAM=39.4GiB | 13.3% done | ETA(epoch): 83s + Train loss: 0.876174 (915.9s) ETA: 4414min + Val loss: 0.864811 [t_0.0-0.2=1.0742 t_0.2-0.4=0.9921 t_0.4-0.6=0.8553 t_0.6-0.8=0.7279 t_0.8-1.0=0.6837] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 40/299 (13% done) --- + [Epoch 40][10/33] loss=0.878995 avg=0.875224 VRAM=39.4GiB | 13.4% done | ETA(epoch): 638s + [Epoch 40][20/33] loss=0.836605 avg=0.876090 VRAM=39.2GiB | 13.5% done | ETA(epoch): 361s + [Epoch 40][30/33] loss=0.845471 avg=0.878994 VRAM=39.4GiB | 13.6% done | ETA(epoch): 83s + Train loss: 0.879824 (916.1s) ETA: 4393min + Val loss: 0.863610 [t_0.0-0.2=1.0647 t_0.2-0.4=1.0124 t_0.4-0.6=0.8593 t_0.6-0.8=0.7486 t_0.8-1.0=0.6893] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 41/299 (14% done) --- + [Epoch 41][10/33] loss=0.813693 avg=0.890336 VRAM=39.4GiB | 13.8% done | ETA(epoch): 639s + [Epoch 41][20/33] loss=0.904436 avg=0.900076 VRAM=39.2GiB | 13.9% done | ETA(epoch): 361s + [Epoch 41][30/33] loss=0.885214 avg=0.882102 VRAM=39.4GiB | 14.0% done | ETA(epoch): 83s + Train loss: 0.880158 (915.4s) ETA: 4372min + Val loss: 0.861687 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0050 t_0.4-0.6=0.8564 t_0.6-0.8=0.7280 t_0.8-1.0=0.6658] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 42/299 (14% done) --- + [Epoch 42][10/33] loss=0.909572 avg=0.868650 VRAM=39.4GiB | 14.1% done | ETA(epoch): 638s + [Epoch 42][20/33] loss=0.927036 avg=0.870016 VRAM=39.2GiB | 14.2% done | ETA(epoch): 361s + [Epoch 42][30/33] loss=0.899307 avg=0.869560 VRAM=39.4GiB | 14.3% done | ETA(epoch): 83s + Train loss: 0.873257 (915.2s) ETA: 4352min + Val loss: 0.860938 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9687 t_0.4-0.6=0.8402 t_0.6-0.8=0.7588 t_0.8-1.0=0.6781] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 43/299 (14% done) --- + [Epoch 43][10/33] loss=0.882293 avg=0.847957 VRAM=39.4GiB | 14.4% done | ETA(epoch): 637s + [Epoch 43][20/33] loss=0.870405 avg=0.855878 VRAM=39.2GiB | 14.5% done | ETA(epoch): 360s + [Epoch 43][30/33] loss=0.775824 avg=0.865763 VRAM=39.4GiB | 14.6% done | ETA(epoch): 83s + Train loss: 0.861632 (915.1s) ETA: 4332min + Val loss: 0.859102 [t_0.0-0.2=1.0855 t_0.2-0.4=1.0244 t_0.4-0.6=0.8068 t_0.6-0.8=0.7453 t_0.8-1.0=0.7092] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 44/299 (15% done) --- + [Epoch 44][10/33] loss=0.900325 avg=0.888251 VRAM=39.4GiB | 14.8% done | ETA(epoch): 637s + [Epoch 44][20/33] loss=0.859217 avg=0.865307 VRAM=39.2GiB | 14.9% done | ETA(epoch): 360s + [Epoch 44][30/33] loss=0.815073 avg=0.863216 VRAM=39.4GiB | 15.0% done | ETA(epoch): 83s + Train loss: 0.862320 (914.4s) ETA: 4311min + Val loss: 0.889658 [t_0.0-0.2=1.0699 t_0.2-0.4=0.9926 t_0.4-0.6=0.8365 t_0.6-0.8=0.7219 t_0.8-1.0=0.7161] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 45/299 (15% done) --- + [Epoch 45][10/33] loss=0.878698 avg=0.873542 VRAM=39.4GiB | 15.1% done | ETA(epoch): 638s + [Epoch 45][20/33] loss=0.936592 avg=0.869347 VRAM=39.2GiB | 15.2% done | ETA(epoch): 361s + [Epoch 45][30/33] loss=0.779277 avg=0.868312 VRAM=39.4GiB | 15.3% done | ETA(epoch): 83s + Train loss: 0.871049 (916.0s) ETA: 4291min + Val loss: 0.872894 [t_0.0-0.2=1.0685 t_0.2-0.4=0.9978 t_0.4-0.6=0.8349 t_0.6-0.8=0.7121 t_0.8-1.0=0.6955] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 46/299 (15% done) --- + [Epoch 46][10/33] loss=0.779621 avg=0.833086 VRAM=39.4GiB | 15.4% done | ETA(epoch): 638s + [Epoch 46][20/33] loss=0.891240 avg=0.859615 VRAM=39.2GiB | 15.5% done | ETA(epoch): 361s + [Epoch 46][30/33] loss=0.846114 avg=0.867198 VRAM=39.4GiB | 15.6% done | ETA(epoch): 83s + Train loss: 0.865639 (915.6s) ETA: 4272min + Val loss: 0.858831 [t_0.0-0.2=1.0790 t_0.2-0.4=1.0040 t_0.4-0.6=0.8666 t_0.6-0.8=0.7320 t_0.8-1.0=0.6776] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 47/299 (16% done) --- + [Epoch 47][10/33] loss=0.779647 avg=0.865090 VRAM=39.4GiB | 15.8% done | ETA(epoch): 638s + [Epoch 47][20/33] loss=0.862967 avg=0.860678 VRAM=39.2GiB | 15.9% done | ETA(epoch): 361s + [Epoch 47][30/33] loss=0.898743 avg=0.868159 VRAM=39.4GiB | 16.0% done | ETA(epoch): 83s + Train loss: 0.868430 (915.6s) ETA: 4252min + Val loss: 0.854146 [t_0.0-0.2=1.0767 t_0.2-0.4=1.0333 t_0.4-0.6=0.8112 t_0.6-0.8=0.7318 t_0.8-1.0=0.6810] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 48/299 (16% done) --- + [Epoch 48][10/33] loss=0.927662 avg=0.874523 VRAM=39.4GiB | 16.1% done | ETA(epoch): 638s + [Epoch 48][20/33] loss=0.876563 avg=0.859734 VRAM=39.2GiB | 16.2% done | ETA(epoch): 361s + [Epoch 48][30/33] loss=0.752923 avg=0.857446 VRAM=39.4GiB | 16.3% done | ETA(epoch): 83s + Train loss: 0.857518 (915.6s) ETA: 4233min + Val loss: 0.902095 [t_0.0-0.2=1.0718 t_0.2-0.4=0.9980 t_0.4-0.6=0.9035 t_0.6-0.8=0.7430 t_0.8-1.0=0.6724] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 49/299 (16% done) --- + [Epoch 49][10/33] loss=0.878948 avg=0.901422 VRAM=39.4GiB | 16.4% done | ETA(epoch): 638s + [Epoch 49][20/33] loss=0.961471 avg=0.901429 VRAM=39.2GiB | 16.5% done | ETA(epoch): 360s + [Epoch 49][30/33] loss=0.819592 avg=0.885457 VRAM=39.4GiB | 16.6% done | ETA(epoch): 83s + Train loss: 0.884044 (914.9s) ETA: 4213min + Val loss: 0.838904 [t_0.0-0.2=1.0832 t_0.2-0.4=0.9912 t_0.4-0.6=0.8720 t_0.6-0.8=0.7252 t_0.8-1.0=0.6843] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0049 (BEST) + Deleted old checkpoint: checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 50/299 (17% done) --- + [Epoch 50][10/33] loss=0.846810 avg=0.867153 VRAM=39.4GiB | 16.8% done | ETA(epoch): 637s + [Epoch 50][20/33] loss=0.863918 avg=0.869613 VRAM=39.2GiB | 16.9% done | ETA(epoch): 360s + [Epoch 50][30/33] loss=0.890696 avg=0.868350 VRAM=39.4GiB | 17.0% done | ETA(epoch): 83s + Train loss: 0.864131 (914.8s) ETA: 4194min + Val loss: 0.911774 [t_0.0-0.2=1.0869 t_0.2-0.4=1.0335 t_0.4-0.6=0.8885 t_0.6-0.8=0.7325 t_0.8-1.0=0.7268] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 51/299 (17% done) --- + [Epoch 51][10/33] loss=0.956324 avg=0.901606 VRAM=39.4GiB | 17.1% done | ETA(epoch): 638s + [Epoch 51][20/33] loss=0.821727 avg=0.896881 VRAM=39.2GiB | 17.2% done | ETA(epoch): 361s + [Epoch 51][30/33] loss=0.873761 avg=0.885438 VRAM=39.4GiB | 17.3% done | ETA(epoch): 83s + Train loss: 0.880395 (915.0s) ETA: 4175min + Val loss: 0.882713 [t_0.0-0.2=1.0819 t_0.2-0.4=0.9867 t_0.4-0.6=0.8265 t_0.6-0.8=0.7162 t_0.8-1.0=0.6931] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 52/299 (17% done) --- + [Epoch 52][10/33] loss=0.887399 avg=0.881355 VRAM=39.4GiB | 17.4% done | ETA(epoch): 637s + [Epoch 52][20/33] loss=0.803558 avg=0.870868 VRAM=39.2GiB | 17.5% done | ETA(epoch): 360s + [Epoch 52][30/33] loss=0.872951 avg=0.868755 VRAM=39.4GiB | 17.6% done | ETA(epoch): 83s + Train loss: 0.869233 (915.3s) ETA: 4156min + Val loss: 0.844620 [t_0.0-0.2=1.0784 t_0.2-0.4=1.0053 t_0.4-0.6=0.8498 t_0.6-0.8=0.7293 t_0.8-1.0=0.6802] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0052 +[MEM @ epoch 52 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 53/299 (18% done) --- + [Epoch 53][10/33] loss=0.799415 avg=0.874805 VRAM=39.4GiB | 17.8% done | ETA(epoch): 638s + [Epoch 53][20/33] loss=0.872358 avg=0.875278 VRAM=39.2GiB | 17.9% done | ETA(epoch): 361s + [Epoch 53][30/33] loss=0.896633 avg=0.880513 VRAM=39.4GiB | 18.0% done | ETA(epoch): 83s + Train loss: 0.879563 (915.5s) ETA: 4137min + Val loss: 0.871223 [t_0.0-0.2=1.0839 t_0.2-0.4=1.0172 t_0.4-0.6=0.8721 t_0.6-0.8=0.7349 t_0.8-1.0=0.6869] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 54/299 (18% done) --- + [Epoch 54][10/33] loss=0.841042 avg=0.873073 VRAM=39.4GiB | 18.1% done | ETA(epoch): 637s + [Epoch 54][20/33] loss=0.954908 avg=0.877728 VRAM=39.2GiB | 18.2% done | ETA(epoch): 361s + [Epoch 54][30/33] loss=0.897844 avg=0.870839 VRAM=39.4GiB | 18.3% done | ETA(epoch): 83s + Train loss: 0.876118 (915.2s) ETA: 4118min + Val loss: 0.918608 [t_0.0-0.2=1.0691 t_0.2-0.4=0.9937 t_0.4-0.6=0.8696 t_0.6-0.8=0.7178 t_0.8-1.0=0.7089] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 55/299 (18% done) --- + [Epoch 55][10/33] loss=0.774571 avg=0.872083 VRAM=39.4GiB | 18.4% done | ETA(epoch): 637s + [Epoch 55][20/33] loss=0.852666 avg=0.862037 VRAM=39.2GiB | 18.5% done | ETA(epoch): 360s + [Epoch 55][30/33] loss=0.941849 avg=0.866126 VRAM=39.4GiB | 18.6% done | ETA(epoch): 83s + Train loss: 0.864822 (914.8s) ETA: 4099min + Val loss: 0.893431 [t_0.0-0.2=1.0608 t_0.2-0.4=0.9823 t_0.4-0.6=0.8301 t_0.6-0.8=0.7297 t_0.8-1.0=0.6588] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 56/299 (19% done) --- + [Epoch 56][10/33] loss=0.785774 avg=0.858664 VRAM=39.4GiB | 18.8% done | ETA(epoch): 637s + [Epoch 56][20/33] loss=0.875225 avg=0.860315 VRAM=39.2GiB | 18.9% done | ETA(epoch): 361s + [Epoch 56][30/33] loss=0.833799 avg=0.872327 VRAM=39.4GiB | 19.0% done | ETA(epoch): 83s + Train loss: 0.871547 (915.1s) ETA: 4080min + Val loss: 0.850686 [t_0.0-0.2=1.0923 t_0.2-0.4=1.0162 t_0.4-0.6=0.8274 t_0.6-0.8=0.7269 t_0.8-1.0=0.6832] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 57/299 (19% done) --- + [Epoch 57][10/33] loss=0.924541 avg=0.886494 VRAM=39.4GiB | 19.1% done | ETA(epoch): 637s + [Epoch 57][20/33] loss=0.878397 avg=0.862997 VRAM=39.2GiB | 19.2% done | ETA(epoch): 360s + [Epoch 57][30/33] loss=0.870971 avg=0.864543 VRAM=39.4GiB | 19.3% done | ETA(epoch): 83s + Train loss: 0.863856 (914.3s) ETA: 4062min + Val loss: 0.874396 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9920 t_0.4-0.6=0.8185 t_0.6-0.8=0.7086 t_0.8-1.0=0.6907] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 58/299 (19% done) --- + [Epoch 58][10/33] loss=0.840296 avg=0.891119 VRAM=39.4GiB | 19.4% done | ETA(epoch): 637s + [Epoch 58][20/33] loss=0.897994 avg=0.882879 VRAM=39.2GiB | 19.5% done | ETA(epoch): 360s + [Epoch 58][30/33] loss=0.886933 avg=0.876569 VRAM=39.4GiB | 19.6% done | ETA(epoch): 83s + Train loss: 0.872874 (914.6s) ETA: 4043min + Val loss: 0.869138 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9794 t_0.4-0.6=0.8568 t_0.6-0.8=0.7491 t_0.8-1.0=0.6725] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 59/299 (20% done) --- + [Epoch 59][10/33] loss=0.850567 avg=0.854805 VRAM=39.4GiB | 19.8% done | ETA(epoch): 637s + [Epoch 59][20/33] loss=0.898498 avg=0.854434 VRAM=39.2GiB | 19.9% done | ETA(epoch): 360s + [Epoch 59][30/33] loss=0.783768 avg=0.843515 VRAM=39.4GiB | 20.0% done | ETA(epoch): 83s + Train loss: 0.846985 (914.6s) ETA: 4025min + Val loss: 0.862048 [t_0.0-0.2=1.0820 t_0.2-0.4=0.9779 t_0.4-0.6=0.8760 t_0.6-0.8=0.7375 t_0.8-1.0=0.6794] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 60/299 (20% done) --- + [MilestoneVis] train_0 step 1980 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 1980 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 1980 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 1980 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 1980 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 1980 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 1980 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 1980 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 1980 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 1980 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 60][10/33] loss=0.840081 avg=0.866798 VRAM=39.4GiB | 20.1% done | ETA(epoch): 2368s + [Epoch 60][20/33] loss=0.884279 avg=0.881345 VRAM=39.2GiB | 20.2% done | ETA(epoch): 849s + [Epoch 60][30/33] loss=0.865667 avg=0.872500 VRAM=39.4GiB | 20.3% done | ETA(epoch): 158s + Train loss: 0.877162 (1666.8s) ETA: 4055min + Val loss: 0.864691 [t_0.0-0.2=1.0766 t_0.2-0.4=1.0031 t_0.4-0.6=0.8253 t_0.6-0.8=0.7243 t_0.8-1.0=0.6762] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0060 + Deleted old checkpoint: checkpoint_epoch_0057 +[MEM @ epoch 60 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 61/299 (20% done) --- + [Epoch 61][10/33] loss=0.865938 avg=0.843656 VRAM=39.4GiB | 20.4% done | ETA(epoch): 637s + [Epoch 61][20/33] loss=0.830404 avg=0.864950 VRAM=39.2GiB | 20.5% done | ETA(epoch): 360s + [Epoch 61][30/33] loss=0.742708 avg=0.868719 VRAM=39.4GiB | 20.6% done | ETA(epoch): 83s + Train loss: 0.866230 (914.2s) ETA: 4036min + Val loss: 0.861488 [t_0.0-0.2=1.0833 t_0.2-0.4=0.9846 t_0.4-0.6=0.8311 t_0.6-0.8=0.7138 t_0.8-1.0=0.6720] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 62/299 (21% done) --- + [Epoch 62][10/33] loss=0.840615 avg=0.873405 VRAM=39.4GiB | 20.8% done | ETA(epoch): 637s + [Epoch 62][20/33] loss=0.861306 avg=0.867153 VRAM=39.2GiB | 20.9% done | ETA(epoch): 360s + [Epoch 62][30/33] loss=0.837666 avg=0.871146 VRAM=39.4GiB | 21.0% done | ETA(epoch): 83s + Train loss: 0.870445 (913.7s) ETA: 4017min + Val loss: 0.876395 [t_0.0-0.2=1.0512 t_0.2-0.4=0.9790 t_0.4-0.6=0.8486 t_0.6-0.8=0.7528 t_0.8-1.0=0.6801] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 63/299 (21% done) --- + [Epoch 63][10/33] loss=0.788973 avg=0.866276 VRAM=39.4GiB | 21.1% done | ETA(epoch): 637s + [Epoch 63][20/33] loss=0.809422 avg=0.861294 VRAM=39.2GiB | 21.2% done | ETA(epoch): 360s + [Epoch 63][30/33] loss=0.975675 avg=0.865931 VRAM=39.4GiB | 21.3% done | ETA(epoch): 83s + Train loss: 0.865553 (914.1s) ETA: 3997min + Val loss: 0.889088 [t_0.0-0.2=1.0742 t_0.2-0.4=1.0405 t_0.4-0.6=0.8225 t_0.6-0.8=0.7400 t_0.8-1.0=0.6605] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 64/299 (21% done) --- + [Epoch 64][10/33] loss=0.880094 avg=0.874135 VRAM=39.4GiB | 21.4% done | ETA(epoch): 637s + [Epoch 64][20/33] loss=0.848292 avg=0.858876 VRAM=39.2GiB | 21.5% done | ETA(epoch): 360s + [Epoch 64][30/33] loss=0.816653 avg=0.854907 VRAM=39.4GiB | 21.6% done | ETA(epoch): 83s + Train loss: 0.856840 (914.6s) ETA: 3978min + Val loss: 0.835074 [t_0.0-0.2=1.0751 t_0.2-0.4=0.9822 t_0.4-0.6=0.8558 t_0.6-0.8=0.7058 t_0.8-1.0=0.6764] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0064 (BEST) + Deleted old checkpoint: checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0061 +[MEM @ epoch 64 end] RAM: 17.2/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 65/299 (22% done) --- + [Epoch 65][10/33] loss=0.821983 avg=0.865267 VRAM=39.4GiB | 21.8% done | ETA(epoch): 637s + [Epoch 65][20/33] loss=0.941617 avg=0.868788 VRAM=39.2GiB | 21.9% done | ETA(epoch): 360s + [Epoch 65][30/33] loss=0.901436 avg=0.868600 VRAM=39.4GiB | 22.0% done | ETA(epoch): 83s + Train loss: 0.869330 (914.7s) ETA: 3959min + Val loss: 0.849979 [t_0.0-0.2=1.0437 t_0.2-0.4=1.0041 t_0.4-0.6=0.8548 t_0.6-0.8=0.7171 t_0.8-1.0=0.6733] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 66/299 (22% done) --- + [Epoch 66][10/33] loss=0.876735 avg=0.869264 VRAM=39.4GiB | 22.1% done | ETA(epoch): 637s + [Epoch 66][20/33] loss=0.837544 avg=0.879102 VRAM=39.2GiB | 22.2% done | ETA(epoch): 360s + [Epoch 66][30/33] loss=0.874277 avg=0.871928 VRAM=39.4GiB | 22.3% done | ETA(epoch): 83s + Train loss: 0.870585 (914.7s) ETA: 3940min + Val loss: 0.883292 [t_0.0-0.2=1.0760 t_0.2-0.4=1.0022 t_0.4-0.6=0.8491 t_0.6-0.8=0.7167 t_0.8-1.0=0.6801] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 67/299 (22% done) --- + [Epoch 67][10/33] loss=0.890436 avg=0.866832 VRAM=39.4GiB | 22.4% done | ETA(epoch): 637s + [Epoch 67][20/33] loss=0.927135 avg=0.881852 VRAM=39.2GiB | 22.5% done | ETA(epoch): 360s + [Epoch 67][30/33] loss=0.974589 avg=0.879230 VRAM=39.4GiB | 22.6% done | ETA(epoch): 83s + Train loss: 0.879293 (914.9s) ETA: 3922min + Val loss: 0.852483 [t_0.0-0.2=1.0863 t_0.2-0.4=1.0091 t_0.4-0.6=0.8424 t_0.6-0.8=0.7304 t_0.8-1.0=0.6691] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0067 +[MEM @ epoch 67 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 68/299 (23% done) --- + [Epoch 68][10/33] loss=0.898617 avg=0.863034 VRAM=39.4GiB | 22.8% done | ETA(epoch): 638s + [Epoch 68][20/33] loss=0.851299 avg=0.861054 VRAM=39.2GiB | 22.9% done | ETA(epoch): 361s + [Epoch 68][30/33] loss=0.825950 avg=0.856347 VRAM=39.4GiB | 23.0% done | ETA(epoch): 83s + Train loss: 0.857911 (915.3s) ETA: 3903min + Val loss: 0.888735 [t_0.0-0.2=1.0763 t_0.2-0.4=0.9996 t_0.4-0.6=0.8233 t_0.6-0.8=0.7032 t_0.8-1.0=0.6610] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 69/299 (23% done) --- + [Epoch 69][10/33] loss=0.770544 avg=0.856099 VRAM=39.4GiB | 23.1% done | ETA(epoch): 637s + [Epoch 69][20/33] loss=0.877627 avg=0.875536 VRAM=39.2GiB | 23.2% done | ETA(epoch): 360s + [Epoch 69][30/33] loss=0.860781 avg=0.863535 VRAM=39.4GiB | 23.3% done | ETA(epoch): 83s + Train loss: 0.863792 (914.2s) ETA: 3884min + Val loss: 0.897045 [t_0.0-0.2=1.0651 t_0.2-0.4=0.9978 t_0.4-0.6=0.9010 t_0.6-0.8=0.7176 t_0.8-1.0=0.6872] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 70/299 (23% done) --- + [Epoch 70][10/33] loss=0.848569 avg=0.886187 VRAM=39.4GiB | 23.4% done | ETA(epoch): 638s + [Epoch 70][20/33] loss=0.834439 avg=0.879426 VRAM=39.2GiB | 23.5% done | ETA(epoch): 360s + [Epoch 70][30/33] loss=0.872770 avg=0.867457 VRAM=39.4GiB | 23.6% done | ETA(epoch): 83s + Train loss: 0.865567 (914.9s) ETA: 3866min + Val loss: 0.885044 [t_0.0-0.2=1.0803 t_0.2-0.4=0.9762 t_0.4-0.6=0.8645 t_0.6-0.8=0.7129 t_0.8-1.0=0.6769] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 17.8/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 71/299 (24% done) --- + [Epoch 71][10/33] loss=0.810214 avg=0.853585 VRAM=39.4GiB | 23.8% done | ETA(epoch): 638s + [Epoch 71][20/33] loss=0.919473 avg=0.864809 VRAM=39.2GiB | 23.9% done | ETA(epoch): 360s + [Epoch 71][30/33] loss=0.786542 avg=0.858253 VRAM=39.4GiB | 24.0% done | ETA(epoch): 83s + Train loss: 0.856068 (914.3s) ETA: 3847min + Val loss: 0.894833 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9694 t_0.4-0.6=0.8637 t_0.6-0.8=0.7135 t_0.8-1.0=0.6807] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 72/299 (24% done) --- + [Epoch 72][10/33] loss=0.819265 avg=0.869186 VRAM=39.4GiB | 24.1% done | ETA(epoch): 637s + [Epoch 72][20/33] loss=0.953773 avg=0.868713 VRAM=39.2GiB | 24.2% done | ETA(epoch): 360s + [Epoch 72][30/33] loss=0.883811 avg=0.867319 VRAM=39.4GiB | 24.3% done | ETA(epoch): 83s + Train loss: 0.864406 (914.2s) ETA: 3828min + Val loss: 0.908526 [t_0.0-0.2=1.0803 t_0.2-0.4=0.9958 t_0.4-0.6=0.8384 t_0.6-0.8=0.7382 t_0.8-1.0=0.6544] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 73/299 (24% done) --- + [Epoch 73][10/33] loss=0.891320 avg=0.866511 VRAM=39.4GiB | 24.4% done | ETA(epoch): 637s + [Epoch 73][20/33] loss=0.902719 avg=0.869063 VRAM=39.2GiB | 24.5% done | ETA(epoch): 360s + [Epoch 73][30/33] loss=0.842212 avg=0.868155 VRAM=39.4GiB | 24.6% done | ETA(epoch): 83s + Train loss: 0.866197 (914.6s) ETA: 3810min + Val loss: 0.846284 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9913 t_0.4-0.6=0.8238 t_0.6-0.8=0.7048 t_0.8-1.0=0.6579] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 74/299 (25% done) --- + [Epoch 74][10/33] loss=0.862882 avg=0.847320 VRAM=39.4GiB | 24.8% done | ETA(epoch): 637s + [Epoch 74][20/33] loss=0.845888 avg=0.851764 VRAM=39.2GiB | 24.9% done | ETA(epoch): 360s + [Epoch 74][30/33] loss=0.908105 avg=0.857395 VRAM=39.4GiB | 25.0% done | ETA(epoch): 83s + Train loss: 0.860171 (914.5s) ETA: 3792min + Val loss: 0.878023 [t_0.0-0.2=1.0619 t_0.2-0.4=0.9551 t_0.4-0.6=0.8959 t_0.6-0.8=0.7316 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 75/299 (25% done) --- + [Epoch 75][10/33] loss=0.847098 avg=0.884475 VRAM=39.4GiB | 25.1% done | ETA(epoch): 637s + [Epoch 75][20/33] loss=0.934226 avg=0.868420 VRAM=39.2GiB | 25.2% done | ETA(epoch): 360s + [Epoch 75][30/33] loss=0.825695 avg=0.865084 VRAM=39.4GiB | 25.3% done | ETA(epoch): 83s + Train loss: 0.864098 (914.8s) ETA: 3773min + Val loss: 0.866899 [t_0.0-0.2=1.0655 t_0.2-0.4=0.9739 t_0.4-0.6=0.8423 t_0.6-0.8=0.7325 t_0.8-1.0=0.6713] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 76/299 (25% done) --- + [Epoch 76][10/33] loss=0.835797 avg=0.833676 VRAM=39.4GiB | 25.4% done | ETA(epoch): 637s + [Epoch 76][20/33] loss=0.804168 avg=0.834271 VRAM=39.2GiB | 25.5% done | ETA(epoch): 360s + [Epoch 76][30/33] loss=0.803662 avg=0.841889 VRAM=39.4GiB | 25.6% done | ETA(epoch): 83s + Train loss: 0.844981 (915.0s) ETA: 3755min + Val loss: 0.850030 [t_0.0-0.2=1.0858 t_0.2-0.4=0.9863 t_0.4-0.6=0.8523 t_0.6-0.8=0.7306 t_0.8-1.0=0.6742] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 77/299 (26% done) --- + [Epoch 77][10/33] loss=0.807765 avg=0.859402 VRAM=39.4GiB | 25.8% done | ETA(epoch): 637s + [Epoch 77][20/33] loss=0.877421 avg=0.849092 VRAM=39.2GiB | 25.9% done | ETA(epoch): 360s + [Epoch 77][30/33] loss=0.886705 avg=0.854837 VRAM=39.4GiB | 26.0% done | ETA(epoch): 83s + Train loss: 0.856779 (914.6s) ETA: 3737min + Val loss: 0.875862 [t_0.0-0.2=1.0686 t_0.2-0.4=0.9980 t_0.4-0.6=0.8904 t_0.6-0.8=0.7325 t_0.8-1.0=0.6656] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 17.8/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 78/299 (26% done) --- + [Epoch 78][10/33] loss=0.890463 avg=0.841102 VRAM=39.4GiB | 26.1% done | ETA(epoch): 637s + [Epoch 78][20/33] loss=0.896602 avg=0.862420 VRAM=39.2GiB | 26.2% done | ETA(epoch): 360s + [Epoch 78][30/33] loss=0.811592 avg=0.861140 VRAM=39.4GiB | 26.3% done | ETA(epoch): 83s + Train loss: 0.858426 (914.8s) ETA: 3719min + Val loss: 0.834977 [t_0.0-0.2=1.0710 t_0.2-0.4=1.0152 t_0.4-0.6=0.7867 t_0.6-0.8=0.7334 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0078 (BEST) + Deleted old checkpoint: checkpoint_epoch_0064 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 17.8/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 79/299 (26% done) --- + [Epoch 79][10/33] loss=0.768569 avg=0.876562 VRAM=39.4GiB | 26.4% done | ETA(epoch): 638s + [Epoch 79][20/33] loss=0.838014 avg=0.862657 VRAM=39.2GiB | 26.5% done | ETA(epoch): 361s + [Epoch 79][30/33] loss=0.859648 avg=0.863005 VRAM=39.4GiB | 26.6% done | ETA(epoch): 83s + Train loss: 0.861391 (915.5s) ETA: 3700min + Val loss: 0.865763 [t_0.0-0.2=1.0661 t_0.2-0.4=1.0020 t_0.4-0.6=0.8648 t_0.6-0.8=0.7107 t_0.8-1.0=0.6704] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 80/299 (27% done) --- + [Epoch 80][10/33] loss=0.831713 avg=0.837824 VRAM=39.4GiB | 26.8% done | ETA(epoch): 638s + [Epoch 80][20/33] loss=0.841826 avg=0.840907 VRAM=39.2GiB | 26.9% done | ETA(epoch): 361s + [Epoch 80][30/33] loss=0.821304 avg=0.856664 VRAM=39.4GiB | 27.0% done | ETA(epoch): 83s + Train loss: 0.863630 (915.5s) ETA: 3682min + Val loss: 0.867997 [t_0.0-0.2=1.0710 t_0.2-0.4=1.0094 t_0.4-0.6=0.9014 t_0.6-0.8=0.7112 t_0.8-1.0=0.6923] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 81/299 (27% done) --- + [Epoch 81][10/33] loss=0.887460 avg=0.877951 VRAM=39.4GiB | 27.1% done | ETA(epoch): 637s + [Epoch 81][20/33] loss=0.949700 avg=0.866431 VRAM=39.2GiB | 27.2% done | ETA(epoch): 361s + [Epoch 81][30/33] loss=0.814738 avg=0.854582 VRAM=39.4GiB | 27.3% done | ETA(epoch): 83s + Train loss: 0.859376 (915.8s) ETA: 3664min + Val loss: 0.871639 [t_0.0-0.2=1.0695 t_0.2-0.4=1.0049 t_0.4-0.6=0.8584 t_0.6-0.8=0.7135 t_0.8-1.0=0.6788] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0081 +[MEM @ epoch 81 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 82/299 (27% done) --- + [Epoch 82][10/33] loss=0.870278 avg=0.847954 VRAM=39.4GiB | 27.4% done | ETA(epoch): 639s + [Epoch 82][20/33] loss=0.871082 avg=0.865380 VRAM=39.2GiB | 27.5% done | ETA(epoch): 361s + [Epoch 82][30/33] loss=0.882925 avg=0.870504 VRAM=39.4GiB | 27.6% done | ETA(epoch): 83s + Train loss: 0.865945 (917.0s) ETA: 3646min + Val loss: 0.852463 [t_0.0-0.2=1.0873 t_0.2-0.4=1.0217 t_0.4-0.6=0.8635 t_0.6-0.8=0.7380 t_0.8-1.0=0.6499] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0082 + Deleted old checkpoint: checkpoint_epoch_0079 +[MEM @ epoch 82 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 83/299 (28% done) --- + [Epoch 83][10/33] loss=0.803024 avg=0.849912 VRAM=39.4GiB | 27.8% done | ETA(epoch): 639s + [Epoch 83][20/33] loss=0.881919 avg=0.868718 VRAM=39.2GiB | 27.9% done | ETA(epoch): 362s + [Epoch 83][30/33] loss=0.981522 avg=0.869980 VRAM=39.4GiB | 28.0% done | ETA(epoch): 83s + Train loss: 0.872402 (917.6s) ETA: 3629min + Val loss: 0.879216 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9896 t_0.4-0.6=0.8679 t_0.6-0.8=0.7376 t_0.8-1.0=0.6641] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 84/299 (28% done) --- + [Epoch 84][10/33] loss=0.831192 avg=0.854909 VRAM=39.4GiB | 28.1% done | ETA(epoch): 639s + [Epoch 84][20/33] loss=0.918429 avg=0.876207 VRAM=39.2GiB | 28.2% done | ETA(epoch): 361s + [Epoch 84][30/33] loss=0.907883 avg=0.874085 VRAM=39.4GiB | 28.3% done | ETA(epoch): 83s + Train loss: 0.868598 (917.2s) ETA: 3611min + Val loss: 0.860432 [t_0.0-0.2=1.0776 t_0.2-0.4=0.9805 t_0.4-0.6=0.8525 t_0.6-0.8=0.7017 t_0.8-1.0=0.6525] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 18.0/188.4 GiB (9.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 85/299 (28% done) --- + [Epoch 85][10/33] loss=0.845362 avg=0.871616 VRAM=39.4GiB | 28.4% done | ETA(epoch): 639s + [Epoch 85][20/33] loss=0.864827 avg=0.854343 VRAM=39.2GiB | 28.5% done | ETA(epoch): 361s + [Epoch 85][30/33] loss=0.900387 avg=0.863670 VRAM=39.4GiB | 28.6% done | ETA(epoch): 83s + Train loss: 0.858851 (916.3s) ETA: 3593min + Val loss: 0.904677 [t_0.0-0.2=1.0724 t_0.2-0.4=1.0061 t_0.4-0.6=0.8381 t_0.6-0.8=0.7492 t_0.8-1.0=0.6735] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 86/299 (29% done) --- + [Epoch 86][10/33] loss=0.939713 avg=0.865274 VRAM=39.4GiB | 28.8% done | ETA(epoch): 638s + [Epoch 86][20/33] loss=0.850853 avg=0.855874 VRAM=39.2GiB | 28.9% done | ETA(epoch): 361s + [Epoch 86][30/33] loss=0.979750 avg=0.860403 VRAM=39.4GiB | 29.0% done | ETA(epoch): 83s + Train loss: 0.861105 (916.1s) ETA: 3575min + Val loss: 0.843926 [t_0.0-0.2=1.0656 t_0.2-0.4=0.9830 t_0.4-0.6=0.8600 t_0.6-0.8=0.7209 t_0.8-1.0=0.6731] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 87/299 (29% done) --- + [Epoch 87][10/33] loss=0.858026 avg=0.844940 VRAM=39.4GiB | 29.1% done | ETA(epoch): 638s + [Epoch 87][20/33] loss=0.805788 avg=0.844427 VRAM=39.2GiB | 29.2% done | ETA(epoch): 361s + [Epoch 87][30/33] loss=0.834522 avg=0.839407 VRAM=39.4GiB | 29.3% done | ETA(epoch): 83s + Train loss: 0.842338 (915.7s) ETA: 3557min + Val loss: 0.855140 [t_0.0-0.2=1.0841 t_0.2-0.4=1.0148 t_0.4-0.6=0.8306 t_0.6-0.8=0.7597 t_0.8-1.0=0.6715] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 17.2/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 88/299 (29% done) --- + [Epoch 88][10/33] loss=0.770946 avg=0.850992 VRAM=39.4GiB | 29.4% done | ETA(epoch): 638s + [Epoch 88][20/33] loss=0.829812 avg=0.827995 VRAM=39.2GiB | 29.5% done | ETA(epoch): 360s + [Epoch 88][30/33] loss=0.798368 avg=0.835850 VRAM=39.4GiB | 29.6% done | ETA(epoch): 83s + Train loss: 0.834377 (915.2s) ETA: 3540min + Val loss: 0.857896 [t_0.0-0.2=1.0731 t_0.2-0.4=0.9922 t_0.4-0.6=0.8227 t_0.6-0.8=0.7160 t_0.8-1.0=0.6419] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 89/299 (30% done) --- + [Epoch 89][10/33] loss=0.899965 avg=0.865001 VRAM=39.4GiB | 29.8% done | ETA(epoch): 638s + [Epoch 89][20/33] loss=0.921425 avg=0.866971 VRAM=39.2GiB | 29.9% done | ETA(epoch): 361s + [Epoch 89][30/33] loss=0.920136 avg=0.859082 VRAM=39.4GiB | 30.0% done | ETA(epoch): 83s + Train loss: 0.859562 (915.6s) ETA: 3522min + Val loss: 0.842111 [t_0.0-0.2=1.0679 t_0.2-0.4=1.0013 t_0.4-0.6=0.8380 t_0.6-0.8=0.7269 t_0.8-1.0=0.6543] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 90/299 (30% done) --- + [MilestoneVis] train_0 step 2970 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 2970 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 2970 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 2970 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 2970 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 2970 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 2970 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 2970 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 2970 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 2970 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 90][10/33] loss=0.908226 avg=0.854681 VRAM=39.4GiB | 30.1% done | ETA(epoch): 2373s + [Epoch 90][20/33] loss=0.907719 avg=0.856985 VRAM=39.2GiB | 30.2% done | ETA(epoch): 851s + [Epoch 90][30/33] loss=0.829676 avg=0.856931 VRAM=39.4GiB | 30.3% done | ETA(epoch): 159s + Train loss: 0.856515 (1669.6s) ETA: 3533min + Val loss: 0.844130 [t_0.0-0.2=1.0505 t_0.2-0.4=0.9932 t_0.4-0.6=0.8331 t_0.6-0.8=0.7251 t_0.8-1.0=0.6611] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 91/299 (30% done) --- + [Epoch 91][10/33] loss=0.784683 avg=0.848378 VRAM=39.4GiB | 30.4% done | ETA(epoch): 638s + [Epoch 91][20/33] loss=0.899849 avg=0.858922 VRAM=39.2GiB | 30.5% done | ETA(epoch): 361s + [Epoch 91][30/33] loss=0.931751 avg=0.859972 VRAM=39.4GiB | 30.6% done | ETA(epoch): 83s + Train loss: 0.861869 (915.7s) ETA: 3515min + Val loss: 0.879303 [t_0.0-0.2=1.0757 t_0.2-0.4=1.0385 t_0.4-0.6=0.8254 t_0.6-0.8=0.7115 t_0.8-1.0=0.6705] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 92/299 (31% done) --- + [Epoch 92][10/33] loss=0.823920 avg=0.862541 VRAM=39.4GiB | 30.8% done | ETA(epoch): 639s + [Epoch 92][20/33] loss=0.852023 avg=0.860360 VRAM=39.2GiB | 30.9% done | ETA(epoch): 361s + [Epoch 92][30/33] loss=0.870165 avg=0.871156 VRAM=39.4GiB | 31.0% done | ETA(epoch): 83s + Train loss: 0.868780 (915.8s) ETA: 3497min + Val loss: 0.857268 [t_0.0-0.2=1.0785 t_0.2-0.4=1.0387 t_0.4-0.6=0.8531 t_0.6-0.8=0.7140 t_0.8-1.0=0.6718] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 17.2/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 93/299 (31% done) --- + [Epoch 93][10/33] loss=0.765618 avg=0.860201 VRAM=39.4GiB | 31.1% done | ETA(epoch): 638s + [Epoch 93][20/33] loss=0.845900 avg=0.857734 VRAM=39.2GiB | 31.2% done | ETA(epoch): 360s + [Epoch 93][30/33] loss=0.915813 avg=0.861003 VRAM=39.4GiB | 31.3% done | ETA(epoch): 83s + Train loss: 0.862379 (915.0s) ETA: 3479min + Val loss: 0.864727 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0059 t_0.4-0.6=0.8347 t_0.6-0.8=0.7241 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 94/299 (31% done) --- + [Epoch 94][10/33] loss=0.883503 avg=0.875561 VRAM=39.4GiB | 31.4% done | ETA(epoch): 637s + [Epoch 94][20/33] loss=0.814291 avg=0.862141 VRAM=39.2GiB | 31.5% done | ETA(epoch): 360s + [Epoch 94][30/33] loss=0.818372 avg=0.854371 VRAM=39.4GiB | 31.6% done | ETA(epoch): 83s + Train loss: 0.855501 (914.6s) ETA: 3461min + Val loss: 0.855588 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0072 t_0.4-0.6=0.8636 t_0.6-0.8=0.7232 t_0.8-1.0=0.6519] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 17.2/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 95/299 (32% done) --- + [Epoch 95][10/33] loss=0.746324 avg=0.857431 VRAM=39.4GiB | 31.8% done | ETA(epoch): 638s + [Epoch 95][20/33] loss=0.911921 avg=0.863548 VRAM=39.2GiB | 31.9% done | ETA(epoch): 361s + [Epoch 95][30/33] loss=0.831045 avg=0.870353 VRAM=39.4GiB | 32.0% done | ETA(epoch): 83s + Train loss: 0.873355 (915.2s) ETA: 3443min + Val loss: 0.858335 [t_0.0-0.2=1.0752 t_0.2-0.4=0.9923 t_0.4-0.6=0.8451 t_0.6-0.8=0.7241 t_0.8-1.0=0.6454] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 96/299 (32% done) --- + [Epoch 96][10/33] loss=0.879206 avg=0.843622 VRAM=39.4GiB | 32.1% done | ETA(epoch): 638s + [Epoch 96][20/33] loss=0.912761 avg=0.859748 VRAM=39.2GiB | 32.2% done | ETA(epoch): 361s + [Epoch 96][30/33] loss=0.791743 avg=0.853898 VRAM=39.4GiB | 32.3% done | ETA(epoch): 83s + Train loss: 0.853118 (916.0s) ETA: 3425min + Val loss: 0.843736 [t_0.0-0.2=1.0763 t_0.2-0.4=0.9748 t_0.4-0.6=0.8435 t_0.6-0.8=0.7219 t_0.8-1.0=0.6611] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 97/299 (32% done) --- + [Epoch 97][10/33] loss=0.865169 avg=0.852028 VRAM=39.4GiB | 32.4% done | ETA(epoch): 638s + [Epoch 97][20/33] loss=0.894833 avg=0.872318 VRAM=39.2GiB | 32.5% done | ETA(epoch): 361s + [Epoch 97][30/33] loss=0.844556 avg=0.873877 VRAM=39.4GiB | 32.6% done | ETA(epoch): 83s + Train loss: 0.874862 (916.3s) ETA: 3407min + Val loss: 0.871958 [t_0.0-0.2=1.0573 t_0.2-0.4=1.0235 t_0.4-0.6=0.8657 t_0.6-0.8=0.7486 t_0.8-1.0=0.6510] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 98/299 (33% done) --- + [Epoch 98][10/33] loss=0.912826 avg=0.895178 VRAM=39.4GiB | 32.8% done | ETA(epoch): 638s + [Epoch 98][20/33] loss=0.849456 avg=0.882312 VRAM=39.2GiB | 32.9% done | ETA(epoch): 361s + [Epoch 98][30/33] loss=0.872506 avg=0.888241 VRAM=39.4GiB | 33.0% done | ETA(epoch): 83s + Train loss: 0.884326 (916.8s) ETA: 3389min + Val loss: 0.868679 [t_0.0-0.2=1.0870 t_0.2-0.4=1.0121 t_0.4-0.6=0.8246 t_0.6-0.8=0.7241 t_0.8-1.0=0.6655] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 99/299 (33% done) --- + [Epoch 99][10/33] loss=0.855433 avg=0.839741 VRAM=39.4GiB | 33.1% done | ETA(epoch): 639s + [Epoch 99][20/33] loss=0.706377 avg=0.836778 VRAM=39.2GiB | 33.2% done | ETA(epoch): 361s + [Epoch 99][30/33] loss=0.962966 avg=0.847992 VRAM=39.4GiB | 33.3% done | ETA(epoch): 83s + Train loss: 0.849721 (917.0s) ETA: 3371min + Val loss: 0.907942 [t_0.0-0.2=1.0668 t_0.2-0.4=0.9970 t_0.4-0.6=0.9425 t_0.6-0.8=0.7050 t_0.8-1.0=0.6614] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 100/299 (33% done) --- + [Epoch 100][10/33] loss=0.929415 avg=0.856980 VRAM=39.4GiB | 33.4% done | ETA(epoch): 639s + [Epoch 100][20/33] loss=0.885409 avg=0.857298 VRAM=39.2GiB | 33.5% done | ETA(epoch): 361s + [Epoch 100][30/33] loss=0.871655 avg=0.860661 VRAM=39.4GiB | 33.6% done | ETA(epoch): 83s + Train loss: 0.862785 (916.7s) ETA: 3353min + Val loss: 0.881463 [t_0.0-0.2=1.0765 t_0.2-0.4=0.9814 t_0.4-0.6=0.8398 t_0.6-0.8=0.7181 t_0.8-1.0=0.6708] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 101/299 (34% done) --- + [Epoch 101][10/33] loss=0.843161 avg=0.841902 VRAM=39.4GiB | 33.8% done | ETA(epoch): 639s + [Epoch 101][20/33] loss=0.827367 avg=0.855724 VRAM=39.2GiB | 33.9% done | ETA(epoch): 361s + [Epoch 101][30/33] loss=0.899434 avg=0.847973 VRAM=39.4GiB | 34.0% done | ETA(epoch): 83s + Train loss: 0.853334 (916.9s) ETA: 3336min + Val loss: 0.860851 [t_0.0-0.2=1.0832 t_0.2-0.4=0.9799 t_0.4-0.6=0.8677 t_0.6-0.8=0.7084 t_0.8-1.0=0.6541] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 102/299 (34% done) --- + [Epoch 102][10/33] loss=0.942960 avg=0.882354 VRAM=39.4GiB | 34.1% done | ETA(epoch): 639s + [Epoch 102][20/33] loss=0.869931 avg=0.858420 VRAM=39.2GiB | 34.2% done | ETA(epoch): 361s + [Epoch 102][30/33] loss=0.935605 avg=0.866807 VRAM=39.4GiB | 34.3% done | ETA(epoch): 83s + Train loss: 0.866745 (917.6s) ETA: 3318min + Val loss: 0.883246 [t_0.0-0.2=1.0646 t_0.2-0.4=1.0115 t_0.4-0.6=0.8459 t_0.6-0.8=0.7356 t_0.8-1.0=0.6706] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 103/299 (34% done) --- + [Epoch 103][10/33] loss=0.935975 avg=0.881363 VRAM=39.4GiB | 34.4% done | ETA(epoch): 639s + [Epoch 103][20/33] loss=0.866791 avg=0.880424 VRAM=39.2GiB | 34.5% done | ETA(epoch): 361s + [Epoch 103][30/33] loss=0.854519 avg=0.870456 VRAM=39.4GiB | 34.6% done | ETA(epoch): 83s + Train loss: 0.868654 (916.9s) ETA: 3300min + Val loss: 0.849511 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9925 t_0.4-0.6=0.8060 t_0.6-0.8=0.7276 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 17.2/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 104/299 (35% done) --- + [Epoch 104][10/33] loss=0.818148 avg=0.852210 VRAM=39.4GiB | 34.8% done | ETA(epoch): 639s + [Epoch 104][20/33] loss=0.909759 avg=0.844274 VRAM=39.2GiB | 34.9% done | ETA(epoch): 361s + [Epoch 104][30/33] loss=0.816713 avg=0.842862 VRAM=39.4GiB | 35.0% done | ETA(epoch): 83s + Train loss: 0.843473 (916.9s) ETA: 3283min + Val loss: 0.848932 [t_0.0-0.2=1.0669 t_0.2-0.4=0.9939 t_0.4-0.6=0.8569 t_0.6-0.8=0.7126 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 105/299 (35% done) --- + [Epoch 105][10/33] loss=0.837330 avg=0.865028 VRAM=39.4GiB | 35.1% done | ETA(epoch): 639s + [Epoch 105][20/33] loss=0.798178 avg=0.848736 VRAM=39.2GiB | 35.2% done | ETA(epoch): 361s + [Epoch 105][30/33] loss=0.871778 avg=0.854232 VRAM=39.4GiB | 35.3% done | ETA(epoch): 83s + Train loss: 0.853344 (916.5s) ETA: 3265min + Val loss: 0.879534 [t_0.0-0.2=1.0729 t_0.2-0.4=0.9611 t_0.4-0.6=0.8602 t_0.6-0.8=0.7143 t_0.8-1.0=0.6635] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 16.2/188.4 GiB (8.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 106/299 (35% done) --- + [Epoch 106][10/33] loss=0.967215 avg=0.863301 VRAM=39.4GiB | 35.4% done | ETA(epoch): 638s + [Epoch 106][20/33] loss=0.809537 avg=0.852669 VRAM=39.2GiB | 35.5% done | ETA(epoch): 361s + [Epoch 106][30/33] loss=0.809085 avg=0.853102 VRAM=39.4GiB | 35.6% done | ETA(epoch): 83s + Train loss: 0.856890 (915.8s) ETA: 3247min + Val loss: 0.859603 [t_0.0-0.2=1.0885 t_0.2-0.4=1.0177 t_0.4-0.6=0.8282 t_0.6-0.8=0.7047 t_0.8-1.0=0.6344] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 16.4/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 107/299 (36% done) --- + [Epoch 107][10/33] loss=0.912743 avg=0.904439 VRAM=39.4GiB | 35.8% done | ETA(epoch): 639s + [Epoch 107][20/33] loss=0.874980 avg=0.882578 VRAM=39.2GiB | 35.9% done | ETA(epoch): 361s + [Epoch 107][30/33] loss=0.930848 avg=0.879641 VRAM=39.4GiB | 36.0% done | ETA(epoch): 83s + Train loss: 0.881353 (916.3s) ETA: 3230min + Val loss: 0.873367 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9962 t_0.4-0.6=0.8219 t_0.6-0.8=0.7265 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 16.5/188.4 GiB (8.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 108/299 (36% done) --- + [Epoch 108][10/33] loss=0.894396 avg=0.827171 VRAM=39.4GiB | 36.1% done | ETA(epoch): 638s + [Epoch 108][20/33] loss=0.952023 avg=0.842748 VRAM=39.2GiB | 36.2% done | ETA(epoch): 361s + [Epoch 108][30/33] loss=0.915148 avg=0.849709 VRAM=39.4GiB | 36.3% done | ETA(epoch): 83s + Train loss: 0.852337 (915.6s) ETA: 3212min + Val loss: 0.852249 [t_0.0-0.2=1.0852 t_0.2-0.4=1.0207 t_0.4-0.6=0.8525 t_0.6-0.8=0.7330 t_0.8-1.0=0.6558] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 16.5/188.4 GiB (8.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 109/299 (36% done) --- + [Epoch 109][10/33] loss=0.925628 avg=0.838808 VRAM=39.4GiB | 36.4% done | ETA(epoch): 638s + [Epoch 109][20/33] loss=0.842253 avg=0.860014 VRAM=39.2GiB | 36.5% done | ETA(epoch): 361s + [Epoch 109][30/33] loss=0.808183 avg=0.856469 VRAM=39.4GiB | 36.6% done | ETA(epoch): 83s + Train loss: 0.855717 (915.8s) ETA: 3194min + Val loss: 0.828015 [t_0.0-0.2=1.0897 t_0.2-0.4=0.9803 t_0.4-0.6=0.8525 t_0.6-0.8=0.7037 t_0.8-1.0=0.6564] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0109 (BEST) + Deleted old checkpoint: checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 16.5/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 110/299 (37% done) --- + [Epoch 110][10/33] loss=0.816038 avg=0.861680 VRAM=39.4GiB | 36.8% done | ETA(epoch): 638s + [Epoch 110][20/33] loss=0.947450 avg=0.857126 VRAM=39.2GiB | 36.9% done | ETA(epoch): 361s + [Epoch 110][30/33] loss=0.807723 avg=0.847741 VRAM=39.4GiB | 37.0% done | ETA(epoch): 83s + Train loss: 0.847729 (915.7s) ETA: 3177min + Val loss: 0.853259 [t_0.0-0.2=1.0904 t_0.2-0.4=0.9899 t_0.4-0.6=0.8531 t_0.6-0.8=0.6882 t_0.8-1.0=0.6459] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 16.5/188.4 GiB (8.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 111/299 (37% done) --- + [Epoch 111][10/33] loss=0.844759 avg=0.861614 VRAM=39.4GiB | 37.1% done | ETA(epoch): 638s + [Epoch 111][20/33] loss=0.903509 avg=0.863430 VRAM=39.2GiB | 37.2% done | ETA(epoch): 361s + [Epoch 111][30/33] loss=0.855855 avg=0.864637 VRAM=39.4GiB | 37.3% done | ETA(epoch): 83s + Train loss: 0.864562 (915.6s) ETA: 3159min + Val loss: 0.864133 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9863 t_0.4-0.6=0.8520 t_0.6-0.8=0.6919 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 16.5/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 112/299 (37% done) --- + [Epoch 112][10/33] loss=0.908823 avg=0.853866 VRAM=39.4GiB | 37.4% done | ETA(epoch): 638s + [Epoch 112][20/33] loss=0.838981 avg=0.860118 VRAM=39.2GiB | 37.5% done | ETA(epoch): 361s + [Epoch 112][30/33] loss=0.845198 avg=0.856477 VRAM=39.4GiB | 37.6% done | ETA(epoch): 83s + Train loss: 0.860448 (915.7s) ETA: 3142min + Val loss: 0.840702 [t_0.0-0.2=1.0688 t_0.2-0.4=0.9932 t_0.4-0.6=0.8784 t_0.6-0.8=0.7175 t_0.8-1.0=0.6515] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0112 +[MEM @ epoch 112 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 113/299 (38% done) --- + [Epoch 113][10/33] loss=0.933959 avg=0.870167 VRAM=39.4GiB | 37.8% done | ETA(epoch): 638s + [Epoch 113][20/33] loss=0.907969 avg=0.868498 VRAM=39.2GiB | 37.9% done | ETA(epoch): 361s + [Epoch 113][30/33] loss=0.885419 avg=0.858813 VRAM=39.4GiB | 38.0% done | ETA(epoch): 83s + Train loss: 0.856299 (915.9s) ETA: 3124min + Val loss: 0.846928 [t_0.0-0.2=1.0677 t_0.2-0.4=1.0091 t_0.4-0.6=0.8246 t_0.6-0.8=0.7176 t_0.8-1.0=0.6484] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0113 + Deleted old checkpoint: checkpoint_epoch_0110 +[MEM @ epoch 113 end] RAM: 16.4/188.4 GiB (8.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 114/299 (38% done) --- + [Epoch 114][10/33] loss=0.911933 avg=0.852390 VRAM=39.4GiB | 38.1% done | ETA(epoch): 638s + [Epoch 114][20/33] loss=0.922501 avg=0.859602 VRAM=39.2GiB | 38.2% done | ETA(epoch): 361s + [Epoch 114][30/33] loss=0.941290 avg=0.855190 VRAM=39.4GiB | 38.3% done | ETA(epoch): 83s + Train loss: 0.856069 (915.9s) ETA: 3107min + Val loss: 0.873775 [t_0.0-0.2=1.0873 t_0.2-0.4=1.0284 t_0.4-0.6=0.8201 t_0.6-0.8=0.7126 t_0.8-1.0=0.6639] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 115/299 (38% done) --- + [Epoch 115][10/33] loss=0.905393 avg=0.844152 VRAM=39.4GiB | 38.4% done | ETA(epoch): 638s + [Epoch 115][20/33] loss=0.898248 avg=0.842451 VRAM=39.2GiB | 38.5% done | ETA(epoch): 361s + [Epoch 115][30/33] loss=0.895436 avg=0.846141 VRAM=39.4GiB | 38.6% done | ETA(epoch): 83s + Train loss: 0.845193 (915.9s) ETA: 3089min + Val loss: 0.863828 [t_0.0-0.2=1.0586 t_0.2-0.4=0.9997 t_0.4-0.6=0.8569 t_0.6-0.8=0.7369 t_0.8-1.0=0.6514] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 116/299 (39% done) --- + [Epoch 116][10/33] loss=0.803246 avg=0.840110 VRAM=39.4GiB | 38.8% done | ETA(epoch): 638s + [Epoch 116][20/33] loss=0.783348 avg=0.840462 VRAM=39.2GiB | 38.9% done | ETA(epoch): 361s + [Epoch 116][30/33] loss=0.827590 avg=0.847580 VRAM=39.4GiB | 39.0% done | ETA(epoch): 83s + Train loss: 0.849578 (915.4s) ETA: 3072min + Val loss: 0.854942 [t_0.0-0.2=1.0797 t_0.2-0.4=1.0061 t_0.4-0.6=0.7895 t_0.6-0.8=0.7012 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 117/299 (39% done) --- + [Epoch 117][10/33] loss=0.846462 avg=0.875077 VRAM=39.4GiB | 39.1% done | ETA(epoch): 637s + [Epoch 117][20/33] loss=0.789421 avg=0.885505 VRAM=39.2GiB | 39.2% done | ETA(epoch): 360s + [Epoch 117][30/33] loss=0.860218 avg=0.867393 VRAM=39.4GiB | 39.3% done | ETA(epoch): 83s + Train loss: 0.865237 (914.3s) ETA: 3054min + Val loss: 0.841231 [t_0.0-0.2=1.0645 t_0.2-0.4=1.0033 t_0.4-0.6=0.8641 t_0.6-0.8=0.7119 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 118/299 (39% done) --- + [Epoch 118][10/33] loss=0.861530 avg=0.841681 VRAM=39.4GiB | 39.4% done | ETA(epoch): 637s + [Epoch 118][20/33] loss=0.726806 avg=0.834771 VRAM=39.2GiB | 39.5% done | ETA(epoch): 360s + [Epoch 118][30/33] loss=0.824914 avg=0.840331 VRAM=39.4GiB | 39.6% done | ETA(epoch): 83s + Train loss: 0.844418 (914.6s) ETA: 3037min + Val loss: 0.868837 [t_0.0-0.2=1.0774 t_0.2-0.4=1.0056 t_0.4-0.6=0.8373 t_0.6-0.8=0.7311 t_0.8-1.0=0.6559] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 16.9/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 119/299 (40% done) --- + [Epoch 119][10/33] loss=0.886189 avg=0.905748 VRAM=39.4GiB | 39.8% done | ETA(epoch): 637s + [Epoch 119][20/33] loss=0.847990 avg=0.875977 VRAM=39.2GiB | 39.9% done | ETA(epoch): 360s + [Epoch 119][30/33] loss=0.822634 avg=0.870175 VRAM=39.4GiB | 40.0% done | ETA(epoch): 83s + Train loss: 0.869917 (914.2s) ETA: 3020min + Val loss: 0.855455 [t_0.0-0.2=1.0823 t_0.2-0.4=1.0221 t_0.4-0.6=0.8635 t_0.6-0.8=0.7043 t_0.8-1.0=0.6400] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 120/299 (40% done) --- + [MilestoneVis] train_0 step 3960 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3960 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3960 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3960 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3960 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3960 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3960 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3960 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3960 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3960 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 120][10/33] loss=0.791923 avg=0.844417 VRAM=39.4GiB | 40.1% done | ETA(epoch): 2372s + [Epoch 120][20/33] loss=0.913363 avg=0.856586 VRAM=39.2GiB | 40.2% done | ETA(epoch): 850s + [Epoch 120][30/33] loss=0.928812 avg=0.856688 VRAM=39.4GiB | 40.3% done | ETA(epoch): 159s + Train loss: 0.857093 (1668.6s) ETA: 3021min + Val loss: 0.867918 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9964 t_0.4-0.6=0.8374 t_0.6-0.8=0.7440 t_0.8-1.0=0.6470] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 121/299 (40% done) --- + [Epoch 121][10/33] loss=0.870023 avg=0.859900 VRAM=39.4GiB | 40.4% done | ETA(epoch): 637s + [Epoch 121][20/33] loss=0.778163 avg=0.851334 VRAM=39.2GiB | 40.5% done | ETA(epoch): 360s + [Epoch 121][30/33] loss=0.826893 avg=0.846594 VRAM=39.4GiB | 40.6% done | ETA(epoch): 83s + Train loss: 0.850639 (913.9s) ETA: 3003min + Val loss: 0.851658 [t_0.0-0.2=1.0664 t_0.2-0.4=0.9877 t_0.4-0.6=0.8488 t_0.6-0.8=0.7211 t_0.8-1.0=0.6552] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 122/299 (41% done) --- + [Epoch 122][10/33] loss=0.835704 avg=0.842876 VRAM=39.4GiB | 40.8% done | ETA(epoch): 637s + [Epoch 122][20/33] loss=0.848194 avg=0.836502 VRAM=39.2GiB | 40.9% done | ETA(epoch): 360s + [Epoch 122][30/33] loss=0.878952 avg=0.851358 VRAM=39.4GiB | 41.0% done | ETA(epoch): 83s + Train loss: 0.853203 (914.8s) ETA: 2985min + Val loss: 0.865939 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9915 t_0.4-0.6=0.8875 t_0.6-0.8=0.7115 t_0.8-1.0=0.6593] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 17.6/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 123/299 (41% done) --- + [Epoch 123][10/33] loss=0.907679 avg=0.844693 VRAM=39.4GiB | 41.1% done | ETA(epoch): 638s + [Epoch 123][20/33] loss=0.814016 avg=0.858232 VRAM=39.2GiB | 41.2% done | ETA(epoch): 361s + [Epoch 123][30/33] loss=0.788714 avg=0.853964 VRAM=39.4GiB | 41.3% done | ETA(epoch): 83s + Train loss: 0.852430 (915.1s) ETA: 2968min + Val loss: 0.888708 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9919 t_0.4-0.6=0.8108 t_0.6-0.8=0.7465 t_0.8-1.0=0.6538] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 124/299 (41% done) --- + [Epoch 124][10/33] loss=0.785323 avg=0.827174 VRAM=39.4GiB | 41.4% done | ETA(epoch): 638s + [Epoch 124][20/33] loss=0.839403 avg=0.835971 VRAM=39.2GiB | 41.5% done | ETA(epoch): 360s + [Epoch 124][30/33] loss=0.822578 avg=0.836589 VRAM=39.4GiB | 41.6% done | ETA(epoch): 83s + Train loss: 0.841599 (915.2s) ETA: 2950min + Val loss: 0.874049 [t_0.0-0.2=1.0590 t_0.2-0.4=1.0305 t_0.4-0.6=0.8626 t_0.6-0.8=0.7339 t_0.8-1.0=0.6519] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 125/299 (42% done) --- + [Epoch 125][10/33] loss=0.908149 avg=0.849819 VRAM=39.4GiB | 41.8% done | ETA(epoch): 638s + [Epoch 125][20/33] loss=0.874547 avg=0.874122 VRAM=39.2GiB | 41.9% done | ETA(epoch): 360s + [Epoch 125][30/33] loss=0.805282 avg=0.863478 VRAM=39.4GiB | 42.0% done | ETA(epoch): 83s + Train loss: 0.865872 (915.1s) ETA: 2933min + Val loss: 0.899569 [t_0.0-0.2=1.0780 t_0.2-0.4=1.0098 t_0.4-0.6=0.8539 t_0.6-0.8=0.6989 t_0.8-1.0=0.6434] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 17.8/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 126/299 (42% done) --- + [Epoch 126][10/33] loss=0.916990 avg=0.888397 VRAM=39.4GiB | 42.1% done | ETA(epoch): 638s + [Epoch 126][20/33] loss=0.836009 avg=0.877806 VRAM=39.2GiB | 42.2% done | ETA(epoch): 361s + [Epoch 126][30/33] loss=0.936953 avg=0.860967 VRAM=39.4GiB | 42.3% done | ETA(epoch): 83s + Train loss: 0.868635 (915.7s) ETA: 2915min + Val loss: 0.842671 [t_0.0-0.2=1.0659 t_0.2-0.4=1.0025 t_0.4-0.6=0.8634 t_0.6-0.8=0.7220 t_0.8-1.0=0.6582] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 127/299 (42% done) --- + [Epoch 127][10/33] loss=0.816214 avg=0.841674 VRAM=39.4GiB | 42.4% done | ETA(epoch): 638s + [Epoch 127][20/33] loss=0.827866 avg=0.848288 VRAM=39.2GiB | 42.5% done | ETA(epoch): 360s + [Epoch 127][30/33] loss=0.846668 avg=0.852262 VRAM=39.4GiB | 42.6% done | ETA(epoch): 83s + Train loss: 0.851092 (915.3s) ETA: 2898min + Val loss: 0.859926 [t_0.0-0.2=1.0813 t_0.2-0.4=0.9837 t_0.4-0.6=0.8419 t_0.6-0.8=0.7186 t_0.8-1.0=0.6457] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 128/299 (43% done) --- + [Epoch 128][10/33] loss=0.855867 avg=0.853670 VRAM=39.4GiB | 42.8% done | ETA(epoch): 637s + [Epoch 128][20/33] loss=0.758045 avg=0.852815 VRAM=39.2GiB | 42.9% done | ETA(epoch): 360s + [Epoch 128][30/33] loss=0.933723 avg=0.863968 VRAM=39.4GiB | 43.0% done | ETA(epoch): 83s + Train loss: 0.860505 (914.7s) ETA: 2880min + Val loss: 0.845510 [t_0.0-0.2=1.0732 t_0.2-0.4=0.9787 t_0.4-0.6=0.8555 t_0.6-0.8=0.7214 t_0.8-1.0=0.6580] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 129/299 (43% done) --- + [Epoch 129][10/33] loss=0.755779 avg=0.877304 VRAM=39.4GiB | 43.1% done | ETA(epoch): 637s + [Epoch 129][20/33] loss=0.907982 avg=0.856004 VRAM=39.2GiB | 43.2% done | ETA(epoch): 360s + [Epoch 129][30/33] loss=0.868613 avg=0.850015 VRAM=39.4GiB | 43.3% done | ETA(epoch): 83s + Train loss: 0.850393 (914.4s) ETA: 2863min + Val loss: 0.830159 [t_0.0-0.2=1.0788 t_0.2-0.4=1.0130 t_0.4-0.6=0.8068 t_0.6-0.8=0.7268 t_0.8-1.0=0.6497] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 17.8/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 130/299 (43% done) --- + [Epoch 130][10/33] loss=0.825751 avg=0.872884 VRAM=39.4GiB | 43.4% done | ETA(epoch): 637s + [Epoch 130][20/33] loss=0.793817 avg=0.845707 VRAM=39.2GiB | 43.5% done | ETA(epoch): 360s + [Epoch 130][30/33] loss=0.836665 avg=0.845193 VRAM=39.4GiB | 43.6% done | ETA(epoch): 83s + Train loss: 0.845497 (914.7s) ETA: 2845min + Val loss: 0.864386 [t_0.0-0.2=1.0762 t_0.2-0.4=0.9813 t_0.4-0.6=0.8413 t_0.6-0.8=0.7207 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 17.8/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 131/299 (44% done) --- + [Epoch 131][10/33] loss=0.832288 avg=0.835476 VRAM=39.4GiB | 43.8% done | ETA(epoch): 637s + [Epoch 131][20/33] loss=0.975546 avg=0.856331 VRAM=39.2GiB | 43.9% done | ETA(epoch): 360s + [Epoch 131][30/33] loss=0.914613 avg=0.849816 VRAM=39.4GiB | 44.0% done | ETA(epoch): 83s + Train loss: 0.848638 (914.8s) ETA: 2828min + Val loss: 0.852829 [t_0.0-0.2=1.0864 t_0.2-0.4=1.0030 t_0.4-0.6=0.8478 t_0.6-0.8=0.7223 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 18.0/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 132/299 (44% done) --- + [Epoch 132][10/33] loss=0.803359 avg=0.856216 VRAM=39.4GiB | 44.1% done | ETA(epoch): 637s + [Epoch 132][20/33] loss=0.913052 avg=0.855098 VRAM=39.2GiB | 44.2% done | ETA(epoch): 360s + [Epoch 132][30/33] loss=0.818819 avg=0.850101 VRAM=39.4GiB | 44.3% done | ETA(epoch): 83s + Train loss: 0.857558 (914.8s) ETA: 2810min + Val loss: 0.857043 [t_0.0-0.2=1.0830 t_0.2-0.4=1.0265 t_0.4-0.6=0.8134 t_0.6-0.8=0.7165 t_0.8-1.0=0.6493] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 17.8/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 133/299 (44% done) --- + [Epoch 133][10/33] loss=0.794477 avg=0.845674 VRAM=39.4GiB | 44.4% done | ETA(epoch): 638s + [Epoch 133][20/33] loss=0.782233 avg=0.850131 VRAM=39.2GiB | 44.5% done | ETA(epoch): 360s + [Epoch 133][30/33] loss=0.803578 avg=0.844587 VRAM=39.4GiB | 44.6% done | ETA(epoch): 83s + Train loss: 0.845874 (914.9s) ETA: 2793min + Val loss: 0.828121 [t_0.0-0.2=1.0843 t_0.2-0.4=0.9941 t_0.4-0.6=0.7981 t_0.6-0.8=0.7276 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 134/299 (45% done) --- + [Epoch 134][10/33] loss=0.855880 avg=0.854388 VRAM=39.4GiB | 44.8% done | ETA(epoch): 637s + [Epoch 134][20/33] loss=0.858613 avg=0.851602 VRAM=39.2GiB | 44.9% done | ETA(epoch): 360s + [Epoch 134][30/33] loss=0.909537 avg=0.857280 VRAM=39.4GiB | 45.0% done | ETA(epoch): 83s + Train loss: 0.858138 (914.6s) ETA: 2775min + Val loss: 0.855081 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9986 t_0.4-0.6=0.8728 t_0.6-0.8=0.7218 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 135/299 (45% done) --- + [Epoch 135][10/33] loss=0.832947 avg=0.870445 VRAM=39.4GiB | 45.1% done | ETA(epoch): 637s + [Epoch 135][20/33] loss=0.848456 avg=0.861626 VRAM=39.2GiB | 45.2% done | ETA(epoch): 360s + [Epoch 135][30/33] loss=0.872700 avg=0.861676 VRAM=39.4GiB | 45.3% done | ETA(epoch): 83s + Train loss: 0.862245 (914.5s) ETA: 2758min + Val loss: 0.855569 [t_0.0-0.2=1.0784 t_0.2-0.4=1.0247 t_0.4-0.6=0.8455 t_0.6-0.8=0.7162 t_0.8-1.0=0.6597] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 136/299 (45% done) --- + [Epoch 136][10/33] loss=0.893262 avg=0.856747 VRAM=39.4GiB | 45.4% done | ETA(epoch): 638s + [Epoch 136][20/33] loss=0.892959 avg=0.871079 VRAM=39.2GiB | 45.5% done | ETA(epoch): 360s + [Epoch 136][30/33] loss=0.741511 avg=0.862084 VRAM=39.4GiB | 45.6% done | ETA(epoch): 83s + Train loss: 0.861025 (914.6s) ETA: 2741min + Val loss: 0.827855 [t_0.0-0.2=1.0900 t_0.2-0.4=0.9741 t_0.4-0.6=0.7975 t_0.6-0.8=0.7189 t_0.8-1.0=0.6502] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0136 (BEST) + Deleted old checkpoint: checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 137/299 (46% done) --- + [Epoch 137][10/33] loss=0.809545 avg=0.851581 VRAM=39.4GiB | 45.8% done | ETA(epoch): 637s + [Epoch 137][20/33] loss=0.857891 avg=0.861879 VRAM=39.2GiB | 45.9% done | ETA(epoch): 360s + [Epoch 137][30/33] loss=0.920149 avg=0.856444 VRAM=39.4GiB | 46.0% done | ETA(epoch): 83s + Train loss: 0.858747 (914.6s) ETA: 2723min + Val loss: 0.850253 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9976 t_0.4-0.6=0.8347 t_0.6-0.8=0.7298 t_0.8-1.0=0.6237] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 17.8/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 138/299 (46% done) --- + [Epoch 138][10/33] loss=0.743101 avg=0.826739 VRAM=39.4GiB | 46.1% done | ETA(epoch): 638s + [Epoch 138][20/33] loss=0.757089 avg=0.833795 VRAM=39.2GiB | 46.2% done | ETA(epoch): 360s + [Epoch 138][30/33] loss=0.795907 avg=0.842660 VRAM=39.4GiB | 46.3% done | ETA(epoch): 83s + Train loss: 0.845668 (915.1s) ETA: 2706min + Val loss: 0.856226 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9837 t_0.4-0.6=0.8136 t_0.6-0.8=0.7117 t_0.8-1.0=0.6452] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 139/299 (46% done) --- + [Epoch 139][10/33] loss=0.868149 avg=0.876207 VRAM=39.4GiB | 46.4% done | ETA(epoch): 637s + [Epoch 139][20/33] loss=0.841033 avg=0.857379 VRAM=39.2GiB | 46.5% done | ETA(epoch): 360s + [Epoch 139][30/33] loss=0.832019 avg=0.852073 VRAM=39.4GiB | 46.6% done | ETA(epoch): 83s + Train loss: 0.849320 (914.8s) ETA: 2689min + Val loss: 0.836539 [t_0.0-0.2=1.0512 t_0.2-0.4=0.9765 t_0.4-0.6=0.8594 t_0.6-0.8=0.7170 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0139 +[MEM @ epoch 139 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 140/299 (47% done) --- + [Epoch 140][10/33] loss=0.839957 avg=0.845803 VRAM=39.4GiB | 46.8% done | ETA(epoch): 637s + [Epoch 140][20/33] loss=0.902023 avg=0.845802 VRAM=39.2GiB | 46.9% done | ETA(epoch): 360s + [Epoch 140][30/33] loss=0.908754 avg=0.848886 VRAM=39.4GiB | 47.0% done | ETA(epoch): 83s + Train loss: 0.850226 (914.9s) ETA: 2671min + Val loss: 0.855017 [t_0.0-0.2=1.0639 t_0.2-0.4=0.9696 t_0.4-0.6=0.8669 t_0.6-0.8=0.7227 t_0.8-1.0=0.6430] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 141/299 (47% done) --- + [Epoch 141][10/33] loss=0.835933 avg=0.851351 VRAM=39.4GiB | 47.1% done | ETA(epoch): 638s + [Epoch 141][20/33] loss=0.830097 avg=0.843998 VRAM=39.2GiB | 47.2% done | ETA(epoch): 360s + [Epoch 141][30/33] loss=0.950144 avg=0.847273 VRAM=39.4GiB | 47.3% done | ETA(epoch): 83s + Train loss: 0.847138 (915.3s) ETA: 2654min + Val loss: 0.861773 [t_0.0-0.2=1.0811 t_0.2-0.4=0.9704 t_0.4-0.6=0.8524 t_0.6-0.8=0.7391 t_0.8-1.0=0.6459] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 142/299 (47% done) --- + [Epoch 142][10/33] loss=0.934104 avg=0.867423 VRAM=39.4GiB | 47.4% done | ETA(epoch): 638s + [Epoch 142][20/33] loss=0.863192 avg=0.868572 VRAM=39.2GiB | 47.5% done | ETA(epoch): 360s + [Epoch 142][30/33] loss=0.903925 avg=0.866024 VRAM=39.4GiB | 47.6% done | ETA(epoch): 83s + Train loss: 0.863131 (915.0s) ETA: 2637min + Val loss: 0.883240 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9652 t_0.4-0.6=0.8518 t_0.6-0.8=0.7084 t_0.8-1.0=0.6314] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 143/299 (48% done) --- + [Epoch 143][10/33] loss=0.871951 avg=0.885802 VRAM=39.4GiB | 47.8% done | ETA(epoch): 637s + [Epoch 143][20/33] loss=0.849555 avg=0.868521 VRAM=39.2GiB | 47.9% done | ETA(epoch): 360s + [Epoch 143][30/33] loss=0.812100 avg=0.861703 VRAM=39.4GiB | 48.0% done | ETA(epoch): 83s + Train loss: 0.860298 (914.7s) ETA: 2619min + Val loss: 0.826829 [t_0.0-0.2=1.0736 t_0.2-0.4=1.0079 t_0.4-0.6=0.8102 t_0.6-0.8=0.7156 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0143 (BEST) + Deleted old checkpoint: checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 17.1/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 144/299 (48% done) --- + [Epoch 144][10/33] loss=0.844123 avg=0.850014 VRAM=39.4GiB | 48.1% done | ETA(epoch): 637s + [Epoch 144][20/33] loss=0.769492 avg=0.851531 VRAM=39.2GiB | 48.2% done | ETA(epoch): 360s + [Epoch 144][30/33] loss=0.921973 avg=0.857700 VRAM=39.4GiB | 48.3% done | ETA(epoch): 83s + Train loss: 0.858415 (914.3s) ETA: 2602min + Val loss: 0.877292 [t_0.0-0.2=1.0677 t_0.2-0.4=1.0187 t_0.4-0.6=0.8165 t_0.6-0.8=0.7537 t_0.8-1.0=0.6601] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 145/299 (48% done) --- + [Epoch 145][10/33] loss=0.882959 avg=0.875277 VRAM=39.4GiB | 48.4% done | ETA(epoch): 637s + [Epoch 145][20/33] loss=0.910264 avg=0.874813 VRAM=39.2GiB | 48.5% done | ETA(epoch): 360s + [Epoch 145][30/33] loss=0.911420 avg=0.867254 VRAM=39.4GiB | 48.6% done | ETA(epoch): 83s + Train loss: 0.869009 (914.5s) ETA: 2585min + Val loss: 0.830011 [t_0.0-0.2=1.0902 t_0.2-0.4=0.9956 t_0.4-0.6=0.8488 t_0.6-0.8=0.7103 t_0.8-1.0=0.6510] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 17.2/188.4 GiB (9.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 146/299 (49% done) --- + [Epoch 146][10/33] loss=0.846242 avg=0.877595 VRAM=39.4GiB | 48.8% done | ETA(epoch): 637s + [Epoch 146][20/33] loss=0.783142 avg=0.857863 VRAM=39.2GiB | 48.9% done | ETA(epoch): 360s + [Epoch 146][30/33] loss=0.741342 avg=0.873611 VRAM=39.4GiB | 49.0% done | ETA(epoch): 83s + Train loss: 0.872275 (914.4s) ETA: 2568min + Val loss: 0.853069 [t_0.0-0.2=1.0440 t_0.2-0.4=0.9956 t_0.4-0.6=0.8088 t_0.6-0.8=0.7234 t_0.8-1.0=0.6229] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0146 +[MEM @ epoch 146 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 147/299 (49% done) --- + [Epoch 147][10/33] loss=0.850797 avg=0.864547 VRAM=39.4GiB | 49.1% done | ETA(epoch): 637s + [Epoch 147][20/33] loss=0.899866 avg=0.856852 VRAM=39.2GiB | 49.2% done | ETA(epoch): 360s + [Epoch 147][30/33] loss=0.859895 avg=0.851703 VRAM=39.4GiB | 49.3% done | ETA(epoch): 83s + Train loss: 0.850046 (914.0s) ETA: 2550min + Val loss: 0.873737 [t_0.0-0.2=1.0741 t_0.2-0.4=0.9863 t_0.4-0.6=0.8763 t_0.6-0.8=0.7170 t_0.8-1.0=0.6423] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 17.6/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 148/299 (49% done) --- + [Epoch 148][10/33] loss=0.830189 avg=0.830826 VRAM=39.4GiB | 49.4% done | ETA(epoch): 637s + [Epoch 148][20/33] loss=0.832446 avg=0.840764 VRAM=39.2GiB | 49.5% done | ETA(epoch): 360s + [Epoch 148][30/33] loss=0.905265 avg=0.842221 VRAM=39.4GiB | 49.6% done | ETA(epoch): 83s + Train loss: 0.845484 (913.7s) ETA: 2533min + Val loss: 0.857621 [t_0.0-0.2=1.0702 t_0.2-0.4=0.9980 t_0.4-0.6=0.8591 t_0.6-0.8=0.7249 t_0.8-1.0=0.6371] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 149/299 (50% done) --- + [Epoch 149][10/33] loss=0.913306 avg=0.862353 VRAM=39.4GiB | 49.8% done | ETA(epoch): 636s + [Epoch 149][20/33] loss=0.868089 avg=0.866853 VRAM=39.2GiB | 49.9% done | ETA(epoch): 360s + [Epoch 149][30/33] loss=0.926537 avg=0.866550 VRAM=39.4GiB | 50.0% done | ETA(epoch): 83s + Train loss: 0.866485 (913.7s) ETA: 2516min + Val loss: 0.873772 [t_0.0-0.2=1.0789 t_0.2-0.4=1.0336 t_0.4-0.6=0.8029 t_0.6-0.8=0.6936 t_0.8-1.0=0.6360] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 150/299 (50% done) --- + [MilestoneVis] train_0 step 4950 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 4950 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 4950 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 4950 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 4950 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 4950 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 4950 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 4950 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 4950 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 4950 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/33] loss=0.836720 avg=0.856486 VRAM=39.4GiB | 50.1% done | ETA(epoch): 2375s + [Epoch 150][20/33] loss=0.904452 avg=0.846601 VRAM=39.2GiB | 50.2% done | ETA(epoch): 851s + [Epoch 150][30/33] loss=0.787200 avg=0.847650 VRAM=39.4GiB | 50.3% done | ETA(epoch): 159s + Train loss: 0.846251 (1669.3s) ETA: 2511min + Val loss: 0.846582 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9953 t_0.4-0.6=0.8278 t_0.6-0.8=0.7235 t_0.8-1.0=0.6440] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 151/299 (50% done) --- + [Epoch 151][10/33] loss=0.786380 avg=0.871705 VRAM=39.4GiB | 50.4% done | ETA(epoch): 637s + [Epoch 151][20/33] loss=0.882204 avg=0.869659 VRAM=39.2GiB | 50.5% done | ETA(epoch): 360s + [Epoch 151][30/33] loss=0.854056 avg=0.873188 VRAM=39.4GiB | 50.6% done | ETA(epoch): 83s + Train loss: 0.875600 (913.3s) ETA: 2494min + Val loss: 0.860970 [t_0.0-0.2=1.0796 t_0.2-0.4=1.0153 t_0.4-0.6=0.8513 t_0.6-0.8=0.6969 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 152/299 (51% done) --- + [Epoch 152][10/33] loss=0.817658 avg=0.870589 VRAM=39.4GiB | 50.8% done | ETA(epoch): 636s + [Epoch 152][20/33] loss=0.840519 avg=0.857252 VRAM=39.2GiB | 50.9% done | ETA(epoch): 360s + [Epoch 152][30/33] loss=0.863500 avg=0.859083 VRAM=39.4GiB | 51.0% done | ETA(epoch): 83s + Train loss: 0.860177 (913.5s) ETA: 2477min + Val loss: 0.835987 [t_0.0-0.2=1.0940 t_0.2-0.4=1.0020 t_0.4-0.6=0.8210 t_0.6-0.8=0.7332 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0152 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 153/299 (51% done) --- + [Epoch 153][10/33] loss=0.841329 avg=0.871771 VRAM=39.4GiB | 51.1% done | ETA(epoch): 637s + [Epoch 153][20/33] loss=0.912095 avg=0.854828 VRAM=39.2GiB | 51.2% done | ETA(epoch): 360s + [Epoch 153][30/33] loss=0.909634 avg=0.859506 VRAM=39.4GiB | 51.3% done | ETA(epoch): 83s + Train loss: 0.857655 (914.3s) ETA: 2459min + Val loss: 0.874391 [t_0.0-0.2=1.0649 t_0.2-0.4=0.9959 t_0.4-0.6=0.8063 t_0.6-0.8=0.7121 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 17.8/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 154/299 (51% done) --- + [Epoch 154][10/33] loss=0.929644 avg=0.864681 VRAM=39.4GiB | 51.4% done | ETA(epoch): 637s + [Epoch 154][20/33] loss=0.941520 avg=0.847554 VRAM=39.2GiB | 51.5% done | ETA(epoch): 360s + [Epoch 154][30/33] loss=0.988810 avg=0.869751 VRAM=39.4GiB | 51.6% done | ETA(epoch): 83s + Train loss: 0.865859 (913.9s) ETA: 2442min + Val loss: 0.832362 [t_0.0-0.2=1.0672 t_0.2-0.4=1.0274 t_0.4-0.6=0.8438 t_0.6-0.8=0.7270 t_0.8-1.0=0.6329] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 17.8/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 155/299 (52% done) --- + [Epoch 155][10/33] loss=0.828494 avg=0.876361 VRAM=39.4GiB | 51.8% done | ETA(epoch): 636s + [Epoch 155][20/33] loss=0.883849 avg=0.871346 VRAM=39.2GiB | 51.9% done | ETA(epoch): 360s + [Epoch 155][30/33] loss=0.852230 avg=0.866099 VRAM=39.4GiB | 52.0% done | ETA(epoch): 83s + Train loss: 0.865441 (912.9s) ETA: 2425min + Val loss: 0.865314 [t_0.0-0.2=1.0799 t_0.2-0.4=1.0077 t_0.4-0.6=0.8723 t_0.6-0.8=0.6900 t_0.8-1.0=0.6520] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0155 + Deleted old checkpoint: checkpoint_epoch_0152 +[MEM @ epoch 155 end] RAM: 18.4/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 156/299 (52% done) --- + [Epoch 156][10/33] loss=0.870942 avg=0.873124 VRAM=39.4GiB | 52.1% done | ETA(epoch): 636s + [Epoch 156][20/33] loss=0.806935 avg=0.860919 VRAM=39.2GiB | 52.2% done | ETA(epoch): 360s + [Epoch 156][30/33] loss=0.894776 avg=0.847456 VRAM=39.4GiB | 52.3% done | ETA(epoch): 83s + Train loss: 0.849164 (913.2s) ETA: 2407min + Val loss: 0.842211 [t_0.0-0.2=1.0562 t_0.2-0.4=1.0249 t_0.4-0.6=0.8564 t_0.6-0.8=0.6987 t_0.8-1.0=0.6509] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 157/299 (52% done) --- + [Epoch 157][10/33] loss=0.844812 avg=0.860583 VRAM=39.4GiB | 52.4% done | ETA(epoch): 637s + [Epoch 157][20/33] loss=0.816628 avg=0.855800 VRAM=39.2GiB | 52.5% done | ETA(epoch): 360s + [Epoch 157][30/33] loss=0.932916 avg=0.866587 VRAM=39.4GiB | 52.6% done | ETA(epoch): 83s + Train loss: 0.860161 (913.7s) ETA: 2390min + Val loss: 0.830004 [t_0.0-0.2=1.0802 t_0.2-0.4=1.0131 t_0.4-0.6=0.8429 t_0.6-0.8=0.7209 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 158/299 (53% done) --- + [Epoch 158][10/33] loss=0.800276 avg=0.884473 VRAM=39.4GiB | 52.8% done | ETA(epoch): 636s + [Epoch 158][20/33] loss=0.788792 avg=0.879334 VRAM=39.2GiB | 52.9% done | ETA(epoch): 360s + [Epoch 158][30/33] loss=0.848552 avg=0.861527 VRAM=39.4GiB | 53.0% done | ETA(epoch): 83s + Train loss: 0.864365 (913.6s) ETA: 2373min + Val loss: 0.866605 [t_0.0-0.2=1.0471 t_0.2-0.4=1.0030 t_0.4-0.6=0.8441 t_0.6-0.8=0.7216 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 18.4/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 159/299 (53% done) --- + [Epoch 159][10/33] loss=0.723369 avg=0.829217 VRAM=39.4GiB | 53.1% done | ETA(epoch): 636s + [Epoch 159][20/33] loss=0.924144 avg=0.844486 VRAM=39.2GiB | 53.2% done | ETA(epoch): 360s + [Epoch 159][30/33] loss=0.820161 avg=0.847159 VRAM=39.4GiB | 53.3% done | ETA(epoch): 83s + Train loss: 0.847361 (913.4s) ETA: 2355min + Val loss: 0.875791 [t_0.0-0.2=1.0774 t_0.2-0.4=1.0206 t_0.4-0.6=0.8426 t_0.6-0.8=0.6631 t_0.8-1.0=0.6296] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 160/299 (53% done) --- + [Epoch 160][10/33] loss=0.947890 avg=0.874096 VRAM=39.4GiB | 53.4% done | ETA(epoch): 637s + [Epoch 160][20/33] loss=0.850515 avg=0.870990 VRAM=39.2GiB | 53.5% done | ETA(epoch): 360s + [Epoch 160][30/33] loss=0.884334 avg=0.863951 VRAM=39.4GiB | 53.6% done | ETA(epoch): 83s + Train loss: 0.864924 (913.5s) ETA: 2338min + Val loss: 0.863517 [t_0.0-0.2=1.0711 t_0.2-0.4=1.0148 t_0.4-0.6=0.8084 t_0.6-0.8=0.7413 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 161/299 (54% done) --- + [Epoch 161][10/33] loss=0.852690 avg=0.850229 VRAM=39.4GiB | 53.8% done | ETA(epoch): 637s + [Epoch 161][20/33] loss=0.900338 avg=0.865642 VRAM=39.2GiB | 53.9% done | ETA(epoch): 360s + [Epoch 161][30/33] loss=0.889057 avg=0.862182 VRAM=39.4GiB | 54.0% done | ETA(epoch): 83s + Train loss: 0.857169 (913.6s) ETA: 2321min + Val loss: 0.836199 [t_0.0-0.2=1.0710 t_0.2-0.4=1.0317 t_0.4-0.6=0.8765 t_0.6-0.8=0.7011 t_0.8-1.0=0.6457] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 162/299 (54% done) --- + [Epoch 162][10/33] loss=0.947875 avg=0.854439 VRAM=39.4GiB | 54.1% done | ETA(epoch): 636s + [Epoch 162][20/33] loss=0.866928 avg=0.854809 VRAM=39.2GiB | 54.2% done | ETA(epoch): 360s + [Epoch 162][30/33] loss=0.822446 avg=0.844645 VRAM=39.4GiB | 54.3% done | ETA(epoch): 83s + Train loss: 0.845479 (913.8s) ETA: 2304min + Val loss: 0.826368 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9692 t_0.4-0.6=0.8064 t_0.6-0.8=0.7273 t_0.8-1.0=0.6376] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0162 (BEST) + Deleted old checkpoint: checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 18.5/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 163/299 (54% done) --- + [Epoch 163][10/33] loss=0.796681 avg=0.875399 VRAM=39.4GiB | 54.4% done | ETA(epoch): 637s + [Epoch 163][20/33] loss=0.817596 avg=0.850302 VRAM=39.2GiB | 54.5% done | ETA(epoch): 360s + [Epoch 163][30/33] loss=0.838099 avg=0.859032 VRAM=39.4GiB | 54.6% done | ETA(epoch): 83s + Train loss: 0.853471 (913.9s) ETA: 2286min + Val loss: 0.872469 [t_0.0-0.2=1.0570 t_0.2-0.4=1.0167 t_0.4-0.6=0.8499 t_0.6-0.8=0.7297 t_0.8-1.0=0.6456] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 18.5/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 164/299 (55% done) --- + [Epoch 164][10/33] loss=0.757852 avg=0.862439 VRAM=39.4GiB | 54.8% done | ETA(epoch): 636s + [Epoch 164][20/33] loss=0.787001 avg=0.844928 VRAM=39.2GiB | 54.9% done | ETA(epoch): 360s + [Epoch 164][30/33] loss=0.846438 avg=0.849308 VRAM=39.4GiB | 55.0% done | ETA(epoch): 83s + Train loss: 0.852118 (913.4s) ETA: 2269min + Val loss: 0.889098 [t_0.0-0.2=1.0661 t_0.2-0.4=0.9643 t_0.4-0.6=0.8649 t_0.6-0.8=0.6943 t_0.8-1.0=0.6449] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 18.5/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 165/299 (55% done) --- + [Epoch 165][10/33] loss=0.859854 avg=0.848971 VRAM=39.4GiB | 55.1% done | ETA(epoch): 637s + [Epoch 165][20/33] loss=0.868919 avg=0.855301 VRAM=39.2GiB | 55.2% done | ETA(epoch): 360s + [Epoch 165][30/33] loss=0.842691 avg=0.856410 VRAM=39.4GiB | 55.3% done | ETA(epoch): 83s + Train loss: 0.854646 (913.6s) ETA: 2252min + Val loss: 0.870766 [t_0.0-0.2=1.0774 t_0.2-0.4=0.9768 t_0.4-0.6=0.8559 t_0.6-0.8=0.7075 t_0.8-1.0=0.6229] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0165 +[MEM @ epoch 165 end] RAM: 18.5/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 166/299 (55% done) --- + [Epoch 166][10/33] loss=0.737966 avg=0.819422 VRAM=39.4GiB | 55.4% done | ETA(epoch): 637s + [Epoch 166][20/33] loss=0.862428 avg=0.836481 VRAM=39.2GiB | 55.5% done | ETA(epoch): 360s + [Epoch 166][30/33] loss=0.831036 avg=0.846532 VRAM=39.4GiB | 55.6% done | ETA(epoch): 83s + Train loss: 0.847026 (913.6s) ETA: 2235min + Val loss: 0.861815 [t_0.0-0.2=1.0932 t_0.2-0.4=1.0024 t_0.4-0.6=0.8580 t_0.6-0.8=0.7155 t_0.8-1.0=0.6543] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 17.6/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 167/299 (56% done) --- + [Epoch 167][10/33] loss=0.876984 avg=0.862430 VRAM=39.4GiB | 55.8% done | ETA(epoch): 636s + [Epoch 167][20/33] loss=0.902686 avg=0.878479 VRAM=39.2GiB | 55.9% done | ETA(epoch): 360s + [Epoch 167][30/33] loss=0.902172 avg=0.870166 VRAM=39.4GiB | 56.0% done | ETA(epoch): 83s + Train loss: 0.868447 (913.5s) ETA: 2218min + Val loss: 0.844431 [t_0.0-0.2=1.0731 t_0.2-0.4=1.0037 t_0.4-0.6=0.8705 t_0.6-0.8=0.7442 t_0.8-1.0=0.6446] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 168/299 (56% done) --- + [Epoch 168][10/33] loss=0.837792 avg=0.860823 VRAM=39.4GiB | 56.1% done | ETA(epoch): 636s + [Epoch 168][20/33] loss=0.833107 avg=0.837137 VRAM=39.2GiB | 56.2% done | ETA(epoch): 360s + [Epoch 168][30/33] loss=0.796474 avg=0.840827 VRAM=39.4GiB | 56.3% done | ETA(epoch): 83s + Train loss: 0.845445 (912.8s) ETA: 2201min + Val loss: 0.856134 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9816 t_0.4-0.6=0.8437 t_0.6-0.8=0.7242 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 169/299 (56% done) --- + [Epoch 169][10/33] loss=0.829749 avg=0.814728 VRAM=39.4GiB | 56.4% done | ETA(epoch): 636s + [Epoch 169][20/33] loss=0.909701 avg=0.859543 VRAM=39.2GiB | 56.5% done | ETA(epoch): 360s + [Epoch 169][30/33] loss=0.883198 avg=0.850239 VRAM=39.4GiB | 56.6% done | ETA(epoch): 83s + Train loss: 0.845812 (913.3s) ETA: 2183min + Val loss: 0.858571 [t_0.0-0.2=1.0818 t_0.2-0.4=0.9921 t_0.4-0.6=0.8439 t_0.6-0.8=0.7089 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 170/299 (57% done) --- + [Epoch 170][10/33] loss=0.881406 avg=0.859893 VRAM=39.4GiB | 56.8% done | ETA(epoch): 637s + [Epoch 170][20/33] loss=0.839587 avg=0.854582 VRAM=39.2GiB | 56.9% done | ETA(epoch): 360s + [Epoch 170][30/33] loss=0.981976 avg=0.862325 VRAM=39.4GiB | 57.0% done | ETA(epoch): 83s + Train loss: 0.860110 (913.7s) ETA: 2166min + Val loss: 0.876811 [t_0.0-0.2=1.0632 t_0.2-0.4=0.9889 t_0.4-0.6=0.8399 t_0.6-0.8=0.7319 t_0.8-1.0=0.6394] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 18.4/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 171/299 (57% done) --- + [Epoch 171][10/33] loss=0.885669 avg=0.858144 VRAM=39.4GiB | 57.1% done | ETA(epoch): 637s + [Epoch 171][20/33] loss=0.841771 avg=0.856423 VRAM=39.2GiB | 57.2% done | ETA(epoch): 360s + [Epoch 171][30/33] loss=0.881786 avg=0.856186 VRAM=39.4GiB | 57.3% done | ETA(epoch): 83s + Train loss: 0.852433 (913.5s) ETA: 2149min + Val loss: 0.888052 [t_0.0-0.2=1.0703 t_0.2-0.4=0.9930 t_0.4-0.6=0.9087 t_0.6-0.8=0.7036 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 172/299 (57% done) --- + [Epoch 172][10/33] loss=0.785527 avg=0.855815 VRAM=39.4GiB | 57.4% done | ETA(epoch): 637s + [Epoch 172][20/33] loss=0.873320 avg=0.843833 VRAM=39.2GiB | 57.5% done | ETA(epoch): 360s + [Epoch 172][30/33] loss=0.868798 avg=0.849587 VRAM=39.4GiB | 57.6% done | ETA(epoch): 83s + Train loss: 0.851274 (913.9s) ETA: 2132min + Val loss: 0.851515 [t_0.0-0.2=1.0768 t_0.2-0.4=0.9656 t_0.4-0.6=0.8590 t_0.6-0.8=0.7258 t_0.8-1.0=0.6498] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 173/299 (58% done) --- + [Epoch 173][10/33] loss=0.784844 avg=0.852611 VRAM=39.4GiB | 57.8% done | ETA(epoch): 636s + [Epoch 173][20/33] loss=0.873919 avg=0.849261 VRAM=39.2GiB | 57.9% done | ETA(epoch): 360s + [Epoch 173][30/33] loss=0.847190 avg=0.851124 VRAM=39.4GiB | 58.0% done | ETA(epoch): 83s + Train loss: 0.849449 (913.3s) ETA: 2115min + Val loss: 0.868594 [t_0.0-0.2=1.0707 t_0.2-0.4=1.0068 t_0.4-0.6=0.8444 t_0.6-0.8=0.7285 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 174/299 (58% done) --- + [Epoch 174][10/33] loss=0.897925 avg=0.878497 VRAM=39.4GiB | 58.1% done | ETA(epoch): 637s + [Epoch 174][20/33] loss=0.891834 avg=0.863874 VRAM=39.2GiB | 58.2% done | ETA(epoch): 360s + [Epoch 174][30/33] loss=0.754886 avg=0.851965 VRAM=39.4GiB | 58.3% done | ETA(epoch): 83s + Train loss: 0.853301 (913.9s) ETA: 2098min + Val loss: 0.899351 [t_0.0-0.2=1.0620 t_0.2-0.4=1.0070 t_0.4-0.6=0.9046 t_0.6-0.8=0.7099 t_0.8-1.0=0.6367] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 18.5/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 175/299 (58% done) --- + [Epoch 175][10/33] loss=0.881433 avg=0.874632 VRAM=39.4GiB | 58.4% done | ETA(epoch): 637s + [Epoch 175][20/33] loss=0.927532 avg=0.871550 VRAM=39.2GiB | 58.5% done | ETA(epoch): 360s + [Epoch 175][30/33] loss=0.871165 avg=0.866583 VRAM=39.4GiB | 58.6% done | ETA(epoch): 83s + Train loss: 0.865308 (914.2s) ETA: 2081min + Val loss: 0.846733 [t_0.0-0.2=1.0690 t_0.2-0.4=0.9977 t_0.4-0.6=0.8891 t_0.6-0.8=0.7131 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 176/299 (59% done) --- + [Epoch 176][10/33] loss=0.840194 avg=0.869156 VRAM=39.4GiB | 58.8% done | ETA(epoch): 637s + [Epoch 176][20/33] loss=0.834339 avg=0.853627 VRAM=39.2GiB | 58.9% done | ETA(epoch): 360s + [Epoch 176][30/33] loss=0.898956 avg=0.851878 VRAM=39.4GiB | 59.0% done | ETA(epoch): 83s + Train loss: 0.854244 (914.5s) ETA: 2064min + Val loss: 0.853497 [t_0.0-0.2=1.0879 t_0.2-0.4=0.9789 t_0.4-0.6=0.8818 t_0.6-0.8=0.7017 t_0.8-1.0=0.6365] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 18.4/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 177/299 (59% done) --- + [Epoch 177][10/33] loss=0.740754 avg=0.845385 VRAM=39.4GiB | 59.1% done | ETA(epoch): 637s + [Epoch 177][20/33] loss=0.806379 avg=0.851842 VRAM=39.2GiB | 59.2% done | ETA(epoch): 360s + [Epoch 177][30/33] loss=0.895230 avg=0.860550 VRAM=39.4GiB | 59.3% done | ETA(epoch): 83s + Train loss: 0.860797 (914.2s) ETA: 2046min + Val loss: 0.832893 [t_0.0-0.2=1.0848 t_0.2-0.4=0.9963 t_0.4-0.6=0.7981 t_0.6-0.8=0.7162 t_0.8-1.0=0.6329] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 18.4/188.4 GiB (9.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 178/299 (59% done) --- + [Epoch 178][10/33] loss=0.926821 avg=0.850852 VRAM=39.4GiB | 59.4% done | ETA(epoch): 637s + [Epoch 178][20/33] loss=0.800912 avg=0.854332 VRAM=39.2GiB | 59.5% done | ETA(epoch): 360s + [Epoch 178][30/33] loss=0.869402 avg=0.855818 VRAM=39.4GiB | 59.6% done | ETA(epoch): 83s + Train loss: 0.853626 (914.2s) ETA: 2029min + Val loss: 0.829013 [t_0.0-0.2=1.0822 t_0.2-0.4=0.9608 t_0.4-0.6=0.8596 t_0.6-0.8=0.7019 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 179/299 (60% done) --- + [Epoch 179][10/33] loss=0.828574 avg=0.852679 VRAM=39.4GiB | 59.8% done | ETA(epoch): 637s + [Epoch 179][20/33] loss=0.926716 avg=0.869277 VRAM=39.2GiB | 59.9% done | ETA(epoch): 360s + [Epoch 179][30/33] loss=0.826651 avg=0.866130 VRAM=39.4GiB | 60.0% done | ETA(epoch): 83s + [MilestoneVis] train_0 step 5939 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 5939 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 5939 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 5939 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 5939 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 5939 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 5939 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 5939 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 5939 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 5939 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + Train loss: 0.867695 (1671.7s) ETA: 2021min + Val loss: 0.869669 [t_0.0-0.2=1.0794 t_0.2-0.4=0.9727 t_0.4-0.6=0.8858 t_0.6-0.8=0.7287 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 180/299 (60% done) --- + [Epoch 180][10/33] loss=0.873407 avg=0.878068 VRAM=39.4GiB | 60.1% done | ETA(epoch): 637s + [Epoch 180][20/33] loss=0.930583 avg=0.880953 VRAM=39.2GiB | 60.2% done | ETA(epoch): 360s + [Epoch 180][30/33] loss=0.797721 avg=0.865727 VRAM=39.4GiB | 60.3% done | ETA(epoch): 83s + Train loss: 0.863381 (914.2s) ETA: 2004min + Val loss: 0.858378 [t_0.0-0.2=1.0807 t_0.2-0.4=1.0023 t_0.4-0.6=0.8595 t_0.6-0.8=0.7214 t_0.8-1.0=0.6231] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 181/299 (60% done) --- + [Epoch 181][10/33] loss=0.813068 avg=0.861088 VRAM=39.4GiB | 60.4% done | ETA(epoch): 637s + [Epoch 181][20/33] loss=0.778394 avg=0.851745 VRAM=39.2GiB | 60.5% done | ETA(epoch): 360s + [Epoch 181][30/33] loss=0.835371 avg=0.856807 VRAM=39.4GiB | 60.6% done | ETA(epoch): 83s + Train loss: 0.857837 (913.6s) ETA: 1986min + Val loss: 0.875184 [t_0.0-0.2=1.0775 t_0.2-0.4=1.0094 t_0.4-0.6=0.8575 t_0.6-0.8=0.7129 t_0.8-1.0=0.6356] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 182/299 (61% done) --- + [Epoch 182][10/33] loss=0.852050 avg=0.883081 VRAM=39.4GiB | 60.8% done | ETA(epoch): 637s + [Epoch 182][20/33] loss=0.896507 avg=0.866143 VRAM=39.2GiB | 60.9% done | ETA(epoch): 360s + [Epoch 182][30/33] loss=0.794324 avg=0.864830 VRAM=39.4GiB | 61.0% done | ETA(epoch): 83s + Train loss: 0.863708 (913.5s) ETA: 1969min + Val loss: 0.851765 [t_0.0-0.2=1.0745 t_0.2-0.4=0.9937 t_0.4-0.6=0.8564 t_0.6-0.8=0.7031 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 183/299 (61% done) --- + [Epoch 183][10/33] loss=0.816739 avg=0.861170 VRAM=39.4GiB | 61.1% done | ETA(epoch): 636s + [Epoch 183][20/33] loss=0.897837 avg=0.864188 VRAM=39.2GiB | 61.2% done | ETA(epoch): 360s + [Epoch 183][30/33] loss=0.804672 avg=0.864506 VRAM=39.4GiB | 61.3% done | ETA(epoch): 83s + Train loss: 0.865667 (913.6s) ETA: 1952min + Val loss: 0.880173 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0282 t_0.4-0.6=0.8764 t_0.6-0.8=0.7039 t_0.8-1.0=0.6245] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 184/299 (61% done) --- + [Epoch 184][10/33] loss=0.847325 avg=0.875542 VRAM=39.4GiB | 61.4% done | ETA(epoch): 637s + [Epoch 184][20/33] loss=0.904478 avg=0.859948 VRAM=39.2GiB | 61.5% done | ETA(epoch): 360s + [Epoch 184][30/33] loss=0.879504 avg=0.861973 VRAM=39.4GiB | 61.6% done | ETA(epoch): 83s + Train loss: 0.858853 (913.5s) ETA: 1935min + Val loss: 0.848924 [t_0.0-0.2=1.0802 t_0.2-0.4=0.9924 t_0.4-0.6=0.8620 t_0.6-0.8=0.7263 t_0.8-1.0=0.6466] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 185/299 (62% done) --- + [Epoch 185][10/33] loss=0.785639 avg=0.868135 VRAM=39.4GiB | 61.8% done | ETA(epoch): 637s + [Epoch 185][20/33] loss=0.847644 avg=0.858833 VRAM=39.2GiB | 61.9% done | ETA(epoch): 360s + [Epoch 185][30/33] loss=0.815352 avg=0.855801 VRAM=39.4GiB | 62.0% done | ETA(epoch): 83s + Train loss: 0.856452 (913.4s) ETA: 1918min + Val loss: 0.864380 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0099 t_0.4-0.6=0.8188 t_0.6-0.8=0.7219 t_0.8-1.0=0.6289] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 186/299 (62% done) --- + [Epoch 186][10/33] loss=0.833120 avg=0.868456 VRAM=39.4GiB | 62.1% done | ETA(epoch): 637s + [Epoch 186][20/33] loss=0.805461 avg=0.859470 VRAM=39.2GiB | 62.2% done | ETA(epoch): 360s + [Epoch 186][30/33] loss=0.910594 avg=0.855808 VRAM=39.4GiB | 62.3% done | ETA(epoch): 83s + Train loss: 0.856811 (913.2s) ETA: 1901min + Val loss: 0.889806 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9871 t_0.4-0.6=0.8260 t_0.6-0.8=0.7072 t_0.8-1.0=0.6506] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 187/299 (62% done) --- + [Epoch 187][10/33] loss=0.882405 avg=0.862027 VRAM=39.4GiB | 62.4% done | ETA(epoch): 636s + [Epoch 187][20/33] loss=0.923636 avg=0.861473 VRAM=39.2GiB | 62.5% done | ETA(epoch): 360s + [Epoch 187][30/33] loss=0.941942 avg=0.870576 VRAM=39.4GiB | 62.6% done | ETA(epoch): 83s + Train loss: 0.867927 (912.9s) ETA: 1884min + Val loss: 0.850715 [t_0.0-0.2=1.0565 t_0.2-0.4=0.9883 t_0.4-0.6=0.8709 t_0.6-0.8=0.7156 t_0.8-1.0=0.6443] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 188/299 (63% done) --- + [Epoch 188][10/33] loss=0.883481 avg=0.859839 VRAM=39.4GiB | 62.8% done | ETA(epoch): 636s + [Epoch 188][20/33] loss=0.818854 avg=0.837345 VRAM=39.2GiB | 62.9% done | ETA(epoch): 360s + [Epoch 188][30/33] loss=0.786480 avg=0.841092 VRAM=39.4GiB | 63.0% done | ETA(epoch): 83s + Train loss: 0.841761 (913.0s) ETA: 1866min + Val loss: 0.875653 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9827 t_0.4-0.6=0.8923 t_0.6-0.8=0.7026 t_0.8-1.0=0.6313] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 189/299 (63% done) --- + [Epoch 189][10/33] loss=0.895803 avg=0.832324 VRAM=39.4GiB | 63.1% done | ETA(epoch): 637s + [Epoch 189][20/33] loss=0.860082 avg=0.844731 VRAM=39.2GiB | 63.2% done | ETA(epoch): 360s + [Epoch 189][30/33] loss=0.880320 avg=0.853630 VRAM=39.4GiB | 63.3% done | ETA(epoch): 83s + Train loss: 0.856910 (914.1s) ETA: 1849min + Val loss: 0.825480 [t_0.0-0.2=1.0802 t_0.2-0.4=1.0086 t_0.4-0.6=0.8403 t_0.6-0.8=0.6916 t_0.8-1.0=0.6253] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0189 (BEST) + Deleted old checkpoint: checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 190/299 (63% done) --- + [Epoch 190][10/33] loss=0.869463 avg=0.827700 VRAM=39.4GiB | 63.4% done | ETA(epoch): 637s + [Epoch 190][20/33] loss=0.846475 avg=0.835724 VRAM=39.2GiB | 63.5% done | ETA(epoch): 360s + [Epoch 190][30/33] loss=0.815547 avg=0.838083 VRAM=39.4GiB | 63.6% done | ETA(epoch): 83s + Train loss: 0.841262 (914.3s) ETA: 1832min + Val loss: 0.847352 [t_0.0-0.2=1.0584 t_0.2-0.4=0.9993 t_0.4-0.6=0.8294 t_0.6-0.8=0.7025 t_0.8-1.0=0.6332] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 191/299 (64% done) --- + [Epoch 191][10/33] loss=0.842536 avg=0.822164 VRAM=39.4GiB | 63.8% done | ETA(epoch): 636s + [Epoch 191][20/33] loss=0.831437 avg=0.827988 VRAM=39.2GiB | 63.9% done | ETA(epoch): 360s + [Epoch 191][30/33] loss=0.842084 avg=0.835885 VRAM=39.4GiB | 64.0% done | ETA(epoch): 83s + Train loss: 0.844903 (913.2s) ETA: 1815min + Val loss: 0.813139 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9919 t_0.4-0.6=0.8252 t_0.6-0.8=0.7236 t_0.8-1.0=0.6574] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0191 (BEST) + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 192/299 (64% done) --- + [Epoch 192][10/33] loss=0.917898 avg=0.882886 VRAM=39.4GiB | 64.1% done | ETA(epoch): 636s + [Epoch 192][20/33] loss=0.964610 avg=0.864808 VRAM=39.2GiB | 64.2% done | ETA(epoch): 360s + [Epoch 192][30/33] loss=0.745221 avg=0.855047 VRAM=39.4GiB | 64.3% done | ETA(epoch): 83s + Train loss: 0.850562 (912.9s) ETA: 1798min + Val loss: 0.824487 [t_0.0-0.2=1.0818 t_0.2-0.4=0.9758 t_0.4-0.6=0.8352 t_0.6-0.8=0.7149 t_0.8-1.0=0.6311] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 193/299 (64% done) --- + [Epoch 193][10/33] loss=0.802013 avg=0.839487 VRAM=39.4GiB | 64.4% done | ETA(epoch): 636s + [Epoch 193][20/33] loss=0.886385 avg=0.847872 VRAM=39.2GiB | 64.5% done | ETA(epoch): 360s + [Epoch 193][30/33] loss=0.743150 avg=0.841396 VRAM=39.4GiB | 64.6% done | ETA(epoch): 83s + Train loss: 0.839812 (912.7s) ETA: 1781min + Val loss: 0.794178 [t_0.0-0.2=1.0821 t_0.2-0.4=0.9793 t_0.4-0.6=0.8408 t_0.6-0.8=0.7176 t_0.8-1.0=0.6196] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0193 (BEST) + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 194/299 (65% done) --- + [Epoch 194][10/33] loss=0.829055 avg=0.834820 VRAM=39.4GiB | 64.8% done | ETA(epoch): 636s + [Epoch 194][20/33] loss=0.884055 avg=0.852581 VRAM=39.2GiB | 64.9% done | ETA(epoch): 359s + [Epoch 194][30/33] loss=0.804817 avg=0.852354 VRAM=39.4GiB | 65.0% done | ETA(epoch): 83s + Train loss: 0.851014 (912.8s) ETA: 1764min + Val loss: 0.847242 [t_0.0-0.2=1.0693 t_0.2-0.4=0.9752 t_0.4-0.6=0.8426 t_0.6-0.8=0.7372 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 195/299 (65% done) --- + [Epoch 195][10/33] loss=0.792940 avg=0.871922 VRAM=39.4GiB | 65.1% done | ETA(epoch): 636s + [Epoch 195][20/33] loss=0.916300 avg=0.870235 VRAM=39.2GiB | 65.2% done | ETA(epoch): 360s + [Epoch 195][30/33] loss=0.900586 avg=0.862241 VRAM=39.4GiB | 65.3% done | ETA(epoch): 83s + Train loss: 0.856403 (912.8s) ETA: 1747min + Val loss: 0.844059 [t_0.0-0.2=1.0715 t_0.2-0.4=0.9828 t_0.4-0.6=0.8482 t_0.6-0.8=0.7191 t_0.8-1.0=0.6381] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 196/299 (65% done) --- + [Epoch 196][10/33] loss=0.804578 avg=0.824342 VRAM=39.4GiB | 65.4% done | ETA(epoch): 637s + [Epoch 196][20/33] loss=0.873040 avg=0.853817 VRAM=39.2GiB | 65.5% done | ETA(epoch): 360s + [Epoch 196][30/33] loss=0.836138 avg=0.851273 VRAM=39.4GiB | 65.6% done | ETA(epoch): 83s + Train loss: 0.852644 (913.6s) ETA: 1730min + Val loss: 0.867891 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0188 t_0.4-0.6=0.8366 t_0.6-0.8=0.7038 t_0.8-1.0=0.6453] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0196 +[MEM @ epoch 196 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 197/299 (66% done) --- + [Epoch 197][10/33] loss=0.859853 avg=0.844337 VRAM=39.4GiB | 65.8% done | ETA(epoch): 636s + [Epoch 197][20/33] loss=0.977103 avg=0.862070 VRAM=39.2GiB | 65.9% done | ETA(epoch): 360s + [Epoch 197][30/33] loss=0.820570 avg=0.849400 VRAM=39.4GiB | 66.0% done | ETA(epoch): 83s + Train loss: 0.845934 (913.0s) ETA: 1713min + Val loss: 0.871864 [t_0.0-0.2=1.0736 t_0.2-0.4=1.0182 t_0.4-0.6=0.8425 t_0.6-0.8=0.7061 t_0.8-1.0=0.6397] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 198/299 (66% done) --- + [Epoch 198][10/33] loss=0.820820 avg=0.827033 VRAM=39.4GiB | 66.1% done | ETA(epoch): 636s + [Epoch 198][20/33] loss=0.878196 avg=0.845241 VRAM=39.2GiB | 66.2% done | ETA(epoch): 360s + [Epoch 198][30/33] loss=0.857709 avg=0.847188 VRAM=39.4GiB | 66.3% done | ETA(epoch): 83s + Train loss: 0.850389 (912.7s) ETA: 1696min + Val loss: 0.884872 [t_0.0-0.2=1.0757 t_0.2-0.4=1.0008 t_0.4-0.6=0.8329 t_0.6-0.8=0.7216 t_0.8-1.0=0.6267] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 199/299 (66% done) --- + [Epoch 199][10/33] loss=0.804264 avg=0.848411 VRAM=39.4GiB | 66.4% done | ETA(epoch): 636s + [Epoch 199][20/33] loss=0.855291 avg=0.853857 VRAM=39.2GiB | 66.5% done | ETA(epoch): 360s + [Epoch 199][30/33] loss=0.902224 avg=0.853364 VRAM=39.4GiB | 66.6% done | ETA(epoch): 83s + Train loss: 0.854838 (912.9s) ETA: 1679min + Val loss: 0.860143 [t_0.0-0.2=1.0811 t_0.2-0.4=1.0120 t_0.4-0.6=0.8198 t_0.6-0.8=0.7049 t_0.8-1.0=0.6401] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 18.9/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 200/299 (67% done) --- + [Epoch 200][10/33] loss=0.865168 avg=0.854004 VRAM=39.4GiB | 66.8% done | ETA(epoch): 636s + [Epoch 200][20/33] loss=0.859141 avg=0.857923 VRAM=39.2GiB | 66.9% done | ETA(epoch): 360s + [Epoch 200][30/33] loss=0.918507 avg=0.861422 VRAM=39.4GiB | 67.0% done | ETA(epoch): 83s + Train loss: 0.860275 (913.2s) ETA: 1662min + Val loss: 0.864966 [t_0.0-0.2=1.0728 t_0.2-0.4=0.9703 t_0.4-0.6=0.8473 t_0.6-0.8=0.7327 t_0.8-1.0=0.6409] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 201/299 (67% done) --- + [Epoch 201][10/33] loss=0.815990 avg=0.839890 VRAM=39.4GiB | 67.1% done | ETA(epoch): 636s + [Epoch 201][20/33] loss=0.855359 avg=0.865947 VRAM=39.2GiB | 67.2% done | ETA(epoch): 360s + [Epoch 201][30/33] loss=0.826277 avg=0.857715 VRAM=39.4GiB | 67.3% done | ETA(epoch): 83s + Train loss: 0.854435 (913.2s) ETA: 1645min + Val loss: 0.870471 [t_0.0-0.2=1.0781 t_0.2-0.4=1.0079 t_0.4-0.6=0.8199 t_0.6-0.8=0.7055 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 202/299 (67% done) --- + [Epoch 202][10/33] loss=0.905639 avg=0.863112 VRAM=39.4GiB | 67.4% done | ETA(epoch): 636s + [Epoch 202][20/33] loss=0.859718 avg=0.876698 VRAM=39.2GiB | 67.5% done | ETA(epoch): 360s + [Epoch 202][30/33] loss=0.872926 avg=0.879623 VRAM=39.4GiB | 67.6% done | ETA(epoch): 83s + Train loss: 0.881542 (913.1s) ETA: 1628min + Val loss: 0.861387 [t_0.0-0.2=1.0768 t_0.2-0.4=1.0253 t_0.4-0.6=0.8213 t_0.6-0.8=0.7051 t_0.8-1.0=0.6328] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 203/299 (68% done) --- + [Epoch 203][10/33] loss=0.931969 avg=0.865004 VRAM=39.4GiB | 67.8% done | ETA(epoch): 636s + [Epoch 203][20/33] loss=0.813334 avg=0.846950 VRAM=39.2GiB | 67.9% done | ETA(epoch): 359s + [Epoch 203][30/33] loss=0.884313 avg=0.845030 VRAM=39.4GiB | 68.0% done | ETA(epoch): 83s + Train loss: 0.846827 (912.5s) ETA: 1611min + Val loss: 0.874132 [t_0.0-0.2=1.0714 t_0.2-0.4=1.0032 t_0.4-0.6=0.8572 t_0.6-0.8=0.7079 t_0.8-1.0=0.6386] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 204/299 (68% done) --- + [Epoch 204][10/33] loss=0.852402 avg=0.846840 VRAM=39.4GiB | 68.1% done | ETA(epoch): 637s + [Epoch 204][20/33] loss=0.879597 avg=0.867708 VRAM=39.2GiB | 68.2% done | ETA(epoch): 360s + [Epoch 204][30/33] loss=0.815215 avg=0.866894 VRAM=39.4GiB | 68.3% done | ETA(epoch): 83s + Train loss: 0.865743 (913.1s) ETA: 1594min + Val loss: 0.868450 [t_0.0-0.2=1.0835 t_0.2-0.4=1.0090 t_0.4-0.6=0.8347 t_0.6-0.8=0.6967 t_0.8-1.0=0.6428] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 19.1/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 205/299 (68% done) --- + [Epoch 205][10/33] loss=0.775148 avg=0.829278 VRAM=39.4GiB | 68.4% done | ETA(epoch): 636s + [Epoch 205][20/33] loss=0.830374 avg=0.854809 VRAM=39.2GiB | 68.5% done | ETA(epoch): 360s + [Epoch 205][30/33] loss=0.869511 avg=0.863008 VRAM=39.4GiB | 68.6% done | ETA(epoch): 83s + Train loss: 0.861749 (913.4s) ETA: 1577min + Val loss: 0.875814 [t_0.0-0.2=1.0570 t_0.2-0.4=1.0335 t_0.4-0.6=0.8221 t_0.6-0.8=0.7177 t_0.8-1.0=0.6311] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 206/299 (69% done) --- + [Epoch 206][10/33] loss=0.817250 avg=0.835017 VRAM=39.4GiB | 68.8% done | ETA(epoch): 636s + [Epoch 206][20/33] loss=0.899152 avg=0.844907 VRAM=39.2GiB | 68.9% done | ETA(epoch): 360s + [Epoch 206][30/33] loss=0.868906 avg=0.847163 VRAM=39.4GiB | 69.0% done | ETA(epoch): 83s + Train loss: 0.847868 (913.3s) ETA: 1560min + Val loss: 0.875900 [t_0.0-0.2=1.0772 t_0.2-0.4=1.0020 t_0.4-0.6=0.8395 t_0.6-0.8=0.6992 t_0.8-1.0=0.6551] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 207/299 (69% done) --- + [Epoch 207][10/33] loss=0.857636 avg=0.864700 VRAM=39.4GiB | 69.1% done | ETA(epoch): 636s + [Epoch 207][20/33] loss=0.780815 avg=0.860254 VRAM=39.2GiB | 69.2% done | ETA(epoch): 359s + [Epoch 207][30/33] loss=0.807936 avg=0.854884 VRAM=39.4GiB | 69.3% done | ETA(epoch): 83s + Train loss: 0.855121 (912.4s) ETA: 1543min + Val loss: 0.861076 [t_0.0-0.2=1.0741 t_0.2-0.4=1.0042 t_0.4-0.6=0.8537 t_0.6-0.8=0.7345 t_0.8-1.0=0.6294] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 208/299 (69% done) --- + [Epoch 208][10/33] loss=0.791578 avg=0.859479 VRAM=39.4GiB | 69.4% done | ETA(epoch): 636s + [Epoch 208][20/33] loss=0.859579 avg=0.854283 VRAM=39.2GiB | 69.5% done | ETA(epoch): 359s + [Epoch 208][30/33] loss=0.893723 avg=0.846868 VRAM=39.4GiB | 69.6% done | ETA(epoch): 83s + Train loss: 0.843961 (912.5s) ETA: 1526min + Val loss: 0.829302 [t_0.0-0.2=1.0716 t_0.2-0.4=1.0060 t_0.4-0.6=0.8417 t_0.6-0.8=0.6994 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 209/299 (70% done) --- + [Epoch 209][10/33] loss=0.918987 avg=0.848310 VRAM=39.4GiB | 69.8% done | ETA(epoch): 636s + [Epoch 209][20/33] loss=0.844900 avg=0.845847 VRAM=39.2GiB | 69.9% done | ETA(epoch): 360s + [Epoch 209][30/33] loss=0.863659 avg=0.846060 VRAM=39.4GiB | 70.0% done | ETA(epoch): 83s + [MilestoneVis] train_0 step 6929 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 6929 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 6929 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 6929 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 6929 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 6929 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 6929 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 6929 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 6929 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 6929 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + Train loss: 0.846003 (1670.8s) ETA: 1514min + Val loss: 0.838448 [t_0.0-0.2=1.0859 t_0.2-0.4=0.9770 t_0.4-0.6=0.8022 t_0.6-0.8=0.7229 t_0.8-1.0=0.6335] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 210/299 (70% done) --- + [Epoch 210][10/33] loss=0.763561 avg=0.875810 VRAM=39.4GiB | 70.1% done | ETA(epoch): 636s + [Epoch 210][20/33] loss=0.807762 avg=0.857960 VRAM=39.2GiB | 70.2% done | ETA(epoch): 359s + [Epoch 210][30/33] loss=0.935338 avg=0.862270 VRAM=39.4GiB | 70.3% done | ETA(epoch): 83s + Train loss: 0.860252 (912.0s) ETA: 1497min + Val loss: 0.894762 [t_0.0-0.2=1.0745 t_0.2-0.4=0.9866 t_0.4-0.6=0.8211 t_0.6-0.8=0.7220 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 19.5/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 211/299 (70% done) --- + [Epoch 211][10/33] loss=0.798521 avg=0.857888 VRAM=39.4GiB | 70.4% done | ETA(epoch): 636s + [Epoch 211][20/33] loss=0.818182 avg=0.873957 VRAM=39.2GiB | 70.5% done | ETA(epoch): 360s + [Epoch 211][30/33] loss=0.945081 avg=0.859577 VRAM=39.4GiB | 70.6% done | ETA(epoch): 83s + Train loss: 0.861740 (912.5s) ETA: 1480min + Val loss: 0.870396 [t_0.0-0.2=1.0781 t_0.2-0.4=1.0034 t_0.4-0.6=0.8404 t_0.6-0.8=0.7060 t_0.8-1.0=0.6705] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 212/299 (71% done) --- + [Epoch 212][10/33] loss=0.898384 avg=0.877222 VRAM=39.4GiB | 70.8% done | ETA(epoch): 636s + [Epoch 212][20/33] loss=0.918786 avg=0.867649 VRAM=39.2GiB | 70.9% done | ETA(epoch): 359s + [Epoch 212][30/33] loss=0.851968 avg=0.859201 VRAM=39.4GiB | 71.0% done | ETA(epoch): 83s + Train loss: 0.859168 (912.4s) ETA: 1463min + Val loss: 0.864296 [t_0.0-0.2=1.0898 t_0.2-0.4=0.9534 t_0.4-0.6=0.8575 t_0.6-0.8=0.7265 t_0.8-1.0=0.6594] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 19.7/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 213/299 (71% done) --- + [Epoch 213][10/33] loss=1.011051 avg=0.879781 VRAM=39.4GiB | 71.1% done | ETA(epoch): 635s + [Epoch 213][20/33] loss=1.008713 avg=0.875260 VRAM=39.2GiB | 71.2% done | ETA(epoch): 359s + [Epoch 213][30/33] loss=0.908181 avg=0.862488 VRAM=39.4GiB | 71.3% done | ETA(epoch): 83s + Train loss: 0.866621 (912.5s) ETA: 1446min + Val loss: 0.823757 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9693 t_0.4-0.6=0.8408 t_0.6-0.8=0.7347 t_0.8-1.0=0.6400] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 214/299 (71% done) --- + [Epoch 214][10/33] loss=0.821154 avg=0.817351 VRAM=39.4GiB | 71.4% done | ETA(epoch): 636s + [Epoch 214][20/33] loss=0.898930 avg=0.830330 VRAM=39.2GiB | 71.5% done | ETA(epoch): 360s + [Epoch 214][30/33] loss=0.830953 avg=0.834832 VRAM=39.4GiB | 71.6% done | ETA(epoch): 83s + Train loss: 0.836732 (913.3s) ETA: 1429min + Val loss: 0.862778 [t_0.0-0.2=1.0676 t_0.2-0.4=0.9863 t_0.4-0.6=0.8453 t_0.6-0.8=0.7283 t_0.8-1.0=0.6689] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 215/299 (72% done) --- + [Epoch 215][10/33] loss=0.804562 avg=0.863025 VRAM=39.4GiB | 71.8% done | ETA(epoch): 636s + [Epoch 215][20/33] loss=0.888388 avg=0.862046 VRAM=39.2GiB | 71.9% done | ETA(epoch): 360s + [Epoch 215][30/33] loss=0.807173 avg=0.861800 VRAM=39.4GiB | 72.0% done | ETA(epoch): 83s + Train loss: 0.861225 (912.7s) ETA: 1412min + Val loss: 0.883319 [t_0.0-0.2=1.0776 t_0.2-0.4=1.0219 t_0.4-0.6=0.8510 t_0.6-0.8=0.7467 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 19.0/188.4 GiB (10.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 216/299 (72% done) --- + [Epoch 216][10/33] loss=0.727024 avg=0.826866 VRAM=39.4GiB | 72.1% done | ETA(epoch): 636s + [Epoch 216][20/33] loss=0.846173 avg=0.842027 VRAM=39.2GiB | 72.2% done | ETA(epoch): 360s + [Epoch 216][30/33] loss=0.911144 avg=0.860628 VRAM=39.4GiB | 72.3% done | ETA(epoch): 83s + Train loss: 0.857462 (913.1s) ETA: 1395min + Val loss: 0.852337 [t_0.0-0.2=1.0633 t_0.2-0.4=1.0456 t_0.4-0.6=0.8416 t_0.6-0.8=0.7053 t_0.8-1.0=0.6369] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 217/299 (72% done) --- + [Epoch 217][10/33] loss=0.874249 avg=0.856451 VRAM=39.4GiB | 72.4% done | ETA(epoch): 636s + [Epoch 217][20/33] loss=0.912192 avg=0.857951 VRAM=39.2GiB | 72.5% done | ETA(epoch): 360s + [Epoch 217][30/33] loss=0.887980 avg=0.860367 VRAM=39.4GiB | 72.6% done | ETA(epoch): 83s + Train loss: 0.862192 (913.3s) ETA: 1378min + Val loss: 0.825741 [t_0.0-0.2=1.0529 t_0.2-0.4=1.0052 t_0.4-0.6=0.8846 t_0.6-0.8=0.7154 t_0.8-1.0=0.6476] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 218/299 (73% done) --- + [Epoch 218][10/33] loss=0.815415 avg=0.824244 VRAM=39.4GiB | 72.8% done | ETA(epoch): 637s + [Epoch 218][20/33] loss=0.875722 avg=0.845863 VRAM=39.2GiB | 72.9% done | ETA(epoch): 360s + [Epoch 218][30/33] loss=0.816763 avg=0.843194 VRAM=39.4GiB | 73.0% done | ETA(epoch): 83s + Train loss: 0.843199 (913.6s) ETA: 1361min + Val loss: 0.863145 [t_0.0-0.2=1.0721 t_0.2-0.4=0.9898 t_0.4-0.6=0.8808 t_0.6-0.8=0.7028 t_0.8-1.0=0.6342] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 219/299 (73% done) --- + [Epoch 219][10/33] loss=0.835849 avg=0.876784 VRAM=39.4GiB | 73.1% done | ETA(epoch): 636s + [Epoch 219][20/33] loss=0.809948 avg=0.845438 VRAM=39.2GiB | 73.2% done | ETA(epoch): 360s + [Epoch 219][30/33] loss=0.958603 avg=0.850549 VRAM=39.4GiB | 73.3% done | ETA(epoch): 83s + Train loss: 0.852034 (913.2s) ETA: 1344min + Val loss: 0.870139 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0034 t_0.4-0.6=0.8273 t_0.6-0.8=0.7174 t_0.8-1.0=0.6378] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 220/299 (73% done) --- + [Epoch 220][10/33] loss=0.866714 avg=0.851512 VRAM=39.4GiB | 73.4% done | ETA(epoch): 636s + [Epoch 220][20/33] loss=0.752532 avg=0.854344 VRAM=39.2GiB | 73.5% done | ETA(epoch): 360s + [Epoch 220][30/33] loss=0.783743 avg=0.858332 VRAM=39.4GiB | 73.6% done | ETA(epoch): 83s + Train loss: 0.857646 (913.3s) ETA: 1327min + Val loss: 0.871601 [t_0.0-0.2=1.0739 t_0.2-0.4=0.9635 t_0.4-0.6=0.8113 t_0.6-0.8=0.7057 t_0.8-1.0=0.6297] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 19.7/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 221/299 (74% done) --- + [Epoch 221][10/33] loss=0.856430 avg=0.855397 VRAM=39.4GiB | 73.8% done | ETA(epoch): 636s + [Epoch 221][20/33] loss=0.846102 avg=0.868691 VRAM=39.2GiB | 73.9% done | ETA(epoch): 360s + [Epoch 221][30/33] loss=0.764292 avg=0.863239 VRAM=39.4GiB | 74.0% done | ETA(epoch): 83s + Train loss: 0.859796 (913.0s) ETA: 1310min + Val loss: 0.841512 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9989 t_0.4-0.6=0.8267 t_0.6-0.8=0.7057 t_0.8-1.0=0.6446] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 222/299 (74% done) --- + [Epoch 222][10/33] loss=0.938744 avg=0.867383 VRAM=39.4GiB | 74.1% done | ETA(epoch): 636s + [Epoch 222][20/33] loss=0.798337 avg=0.858734 VRAM=39.2GiB | 74.2% done | ETA(epoch): 360s + [Epoch 222][30/33] loss=0.860174 avg=0.868944 VRAM=39.4GiB | 74.3% done | ETA(epoch): 83s + Train loss: 0.865162 (912.5s) ETA: 1293min + Val loss: 0.885412 [t_0.0-0.2=1.0710 t_0.2-0.4=0.9771 t_0.4-0.6=0.8632 t_0.6-0.8=0.7410 t_0.8-1.0=0.6396] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 19.5/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 223/299 (74% done) --- + [Epoch 223][10/33] loss=0.800208 avg=0.842198 VRAM=39.4GiB | 74.4% done | ETA(epoch): 636s + [Epoch 223][20/33] loss=0.905139 avg=0.843355 VRAM=39.2GiB | 74.5% done | ETA(epoch): 360s + [Epoch 223][30/33] loss=0.737142 avg=0.845555 VRAM=39.4GiB | 74.6% done | ETA(epoch): 83s + Train loss: 0.848038 (912.8s) ETA: 1276min + Val loss: 0.840850 [t_0.0-0.2=1.0811 t_0.2-0.4=0.9965 t_0.4-0.6=0.8078 t_0.6-0.8=0.7100 t_0.8-1.0=0.6448] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 224/299 (75% done) --- + [Epoch 224][10/33] loss=0.935197 avg=0.856433 VRAM=39.4GiB | 74.8% done | ETA(epoch): 637s + [Epoch 224][20/33] loss=0.923664 avg=0.859387 VRAM=39.2GiB | 74.9% done | ETA(epoch): 360s + [Epoch 224][30/33] loss=0.816956 avg=0.873645 VRAM=39.4GiB | 75.0% done | ETA(epoch): 83s + Train loss: 0.872955 (913.4s) ETA: 1259min + Val loss: 0.866142 [t_0.0-0.2=1.0760 t_0.2-0.4=1.0087 t_0.4-0.6=0.8621 t_0.6-0.8=0.7115 t_0.8-1.0=0.6464] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 19.7/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 225/299 (75% done) --- + [Epoch 225][10/33] loss=0.914152 avg=0.861644 VRAM=39.4GiB | 75.1% done | ETA(epoch): 636s + [Epoch 225][20/33] loss=0.832807 avg=0.839850 VRAM=39.2GiB | 75.2% done | ETA(epoch): 360s + [Epoch 225][30/33] loss=0.936686 avg=0.839186 VRAM=39.4GiB | 75.3% done | ETA(epoch): 83s + Train loss: 0.838106 (912.9s) ETA: 1242min + Val loss: 0.818552 [t_0.0-0.2=1.0672 t_0.2-0.4=0.9768 t_0.4-0.6=0.8448 t_0.6-0.8=0.7120 t_0.8-1.0=0.6455] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 19.7/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 226/299 (75% done) --- + [Epoch 226][10/33] loss=0.793901 avg=0.855744 VRAM=39.4GiB | 75.4% done | ETA(epoch): 636s + [Epoch 226][20/33] loss=0.788366 avg=0.855166 VRAM=39.2GiB | 75.5% done | ETA(epoch): 359s + [Epoch 226][30/33] loss=0.876119 avg=0.855763 VRAM=39.4GiB | 75.6% done | ETA(epoch): 83s + Train loss: 0.855317 (912.1s) ETA: 1226min + Val loss: 0.813553 [t_0.0-0.2=1.0757 t_0.2-0.4=0.9458 t_0.4-0.6=0.8176 t_0.6-0.8=0.7051 t_0.8-1.0=0.6351] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 227/299 (76% done) --- + [Epoch 227][10/33] loss=0.765770 avg=0.833400 VRAM=39.4GiB | 75.8% done | ETA(epoch): 635s + [Epoch 227][20/33] loss=0.866786 avg=0.838233 VRAM=39.2GiB | 75.9% done | ETA(epoch): 359s + [Epoch 227][30/33] loss=0.874322 avg=0.836968 VRAM=39.4GiB | 76.0% done | ETA(epoch): 83s + Train loss: 0.835819 (912.0s) ETA: 1209min + Val loss: 0.808146 [t_0.0-0.2=1.0528 t_0.2-0.4=0.9936 t_0.4-0.6=0.8429 t_0.6-0.8=0.7261 t_0.8-1.0=0.6279] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 228/299 (76% done) --- + [Epoch 228][10/33] loss=0.803201 avg=0.878450 VRAM=39.4GiB | 76.1% done | ETA(epoch): 636s + [Epoch 228][20/33] loss=0.839764 avg=0.867530 VRAM=39.2GiB | 76.2% done | ETA(epoch): 360s + [Epoch 228][30/33] loss=0.848846 avg=0.866501 VRAM=39.4GiB | 76.3% done | ETA(epoch): 83s + Train loss: 0.863726 (912.7s) ETA: 1192min + Val loss: 0.842906 [t_0.0-0.2=1.0773 t_0.2-0.4=0.9893 t_0.4-0.6=0.8584 t_0.6-0.8=0.7219 t_0.8-1.0=0.6361] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 19.7/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 229/299 (76% done) --- + [Epoch 229][10/33] loss=0.903438 avg=0.861269 VRAM=39.4GiB | 76.4% done | ETA(epoch): 635s + [Epoch 229][20/33] loss=0.937996 avg=0.860557 VRAM=39.2GiB | 76.5% done | ETA(epoch): 359s + [Epoch 229][30/33] loss=0.792468 avg=0.854565 VRAM=39.4GiB | 76.6% done | ETA(epoch): 83s + Train loss: 0.857338 (911.7s) ETA: 1175min + Val loss: 0.853390 [t_0.0-0.2=1.0811 t_0.2-0.4=1.0328 t_0.4-0.6=0.8263 t_0.6-0.8=0.7065 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 230/299 (77% done) --- + [Epoch 230][10/33] loss=0.770509 avg=0.875818 VRAM=39.4GiB | 76.8% done | ETA(epoch): 636s + [Epoch 230][20/33] loss=0.823023 avg=0.863254 VRAM=39.2GiB | 76.9% done | ETA(epoch): 359s + [Epoch 230][30/33] loss=0.866958 avg=0.858105 VRAM=39.4GiB | 77.0% done | ETA(epoch): 83s + Train loss: 0.855895 (912.3s) ETA: 1158min + Val loss: 0.843878 [t_0.0-0.2=1.0632 t_0.2-0.4=0.9903 t_0.4-0.6=0.7812 t_0.6-0.8=0.7235 t_0.8-1.0=0.6483] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 19.9/188.4 GiB (10.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 231/299 (77% done) --- + [Epoch 231][10/33] loss=0.752854 avg=0.826421 VRAM=39.4GiB | 77.1% done | ETA(epoch): 636s + [Epoch 231][20/33] loss=0.808344 avg=0.841821 VRAM=39.2GiB | 77.2% done | ETA(epoch): 359s + [Epoch 231][30/33] loss=0.825153 avg=0.842981 VRAM=39.4GiB | 77.3% done | ETA(epoch): 83s + Train loss: 0.844761 (912.3s) ETA: 1141min + Val loss: 0.837873 [t_0.0-0.2=1.0913 t_0.2-0.4=0.9758 t_0.4-0.6=0.8303 t_0.6-0.8=0.7193 t_0.8-1.0=0.6340] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0231 + Deleted old checkpoint: checkpoint_epoch_0228 +[MEM @ epoch 231 end] RAM: 19.7/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 232/299 (77% done) --- + [Epoch 232][10/33] loss=0.877549 avg=0.843742 VRAM=39.4GiB | 77.4% done | ETA(epoch): 636s + [Epoch 232][20/33] loss=0.838178 avg=0.857424 VRAM=39.2GiB | 77.5% done | ETA(epoch): 359s + [Epoch 232][30/33] loss=0.839329 avg=0.853801 VRAM=39.4GiB | 77.6% done | ETA(epoch): 83s + Train loss: 0.854121 (912.7s) ETA: 1124min + Val loss: 0.876461 [t_0.0-0.2=1.0788 t_0.2-0.4=0.9766 t_0.4-0.6=0.8470 t_0.6-0.8=0.7211 t_0.8-1.0=0.6474] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0232 + Deleted old checkpoint: checkpoint_epoch_0229 +[MEM @ epoch 232 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 233/299 (78% done) --- + [Epoch 233][10/33] loss=0.836139 avg=0.846141 VRAM=39.4GiB | 77.8% done | ETA(epoch): 636s + [Epoch 233][20/33] loss=0.841822 avg=0.850518 VRAM=39.2GiB | 77.9% done | ETA(epoch): 359s + [Epoch 233][30/33] loss=0.789735 avg=0.843501 VRAM=39.4GiB | 78.0% done | ETA(epoch): 83s + Train loss: 0.842655 (912.5s) ETA: 1107min + Val loss: 0.826291 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9937 t_0.4-0.6=0.8526 t_0.6-0.8=0.6849 t_0.8-1.0=0.6327] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0233 + Deleted old checkpoint: checkpoint_epoch_0230 +[MEM @ epoch 233 end] RAM: 19.7/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 234/299 (78% done) --- + [Epoch 234][10/33] loss=0.827851 avg=0.837103 VRAM=39.4GiB | 78.1% done | ETA(epoch): 636s + [Epoch 234][20/33] loss=0.863576 avg=0.843763 VRAM=39.2GiB | 78.2% done | ETA(epoch): 359s + [Epoch 234][30/33] loss=0.830081 avg=0.842639 VRAM=39.4GiB | 78.3% done | ETA(epoch): 83s + Train loss: 0.843389 (912.5s) ETA: 1090min + Val loss: 0.860621 [t_0.0-0.2=1.0692 t_0.2-0.4=0.9561 t_0.4-0.6=0.8569 t_0.6-0.8=0.7164 t_0.8-1.0=0.6330] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0234 + Deleted old checkpoint: checkpoint_epoch_0231 +[MEM @ epoch 234 end] RAM: 19.6/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 235/299 (78% done) --- + [Epoch 235][10/33] loss=0.750130 avg=0.831644 VRAM=39.4GiB | 78.4% done | ETA(epoch): 636s + [Epoch 235][20/33] loss=0.859393 avg=0.846369 VRAM=39.2GiB | 78.5% done | ETA(epoch): 360s + [Epoch 235][30/33] loss=0.869326 avg=0.855653 VRAM=39.4GiB | 78.6% done | ETA(epoch): 83s + Train loss: 0.856774 (913.1s) ETA: 1073min + Val loss: 0.853798 [t_0.0-0.2=1.0745 t_0.2-0.4=1.0026 t_0.4-0.6=0.8178 t_0.6-0.8=0.7297 t_0.8-1.0=0.6183] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0235 + Deleted old checkpoint: checkpoint_epoch_0232 +[MEM @ epoch 235 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 236/299 (79% done) --- + [Epoch 236][10/33] loss=0.855711 avg=0.849423 VRAM=39.4GiB | 78.8% done | ETA(epoch): 636s + [Epoch 236][20/33] loss=0.890298 avg=0.857522 VRAM=39.2GiB | 78.9% done | ETA(epoch): 360s + [Epoch 236][30/33] loss=0.864108 avg=0.859923 VRAM=39.4GiB | 79.0% done | ETA(epoch): 83s + Train loss: 0.857190 (913.3s) ETA: 1056min + Val loss: 0.870474 [t_0.0-0.2=1.0622 t_0.2-0.4=1.0159 t_0.4-0.6=0.8467 t_0.6-0.8=0.7034 t_0.8-1.0=0.6522] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0236 + Deleted old checkpoint: checkpoint_epoch_0233 +[MEM @ epoch 236 end] RAM: 19.7/188.4 GiB (10.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 237/299 (79% done) --- + [Epoch 237][10/33] loss=0.803938 avg=0.829079 VRAM=39.4GiB | 79.1% done | ETA(epoch): 636s + [Epoch 237][20/33] loss=0.809019 avg=0.840752 VRAM=39.2GiB | 79.2% done | ETA(epoch): 360s + [Epoch 237][30/33] loss=0.852144 avg=0.837821 VRAM=39.4GiB | 79.3% done | ETA(epoch): 83s + Train loss: 0.844284 (912.8s) ETA: 1039min + Val loss: 0.829709 [t_0.0-0.2=1.0763 t_0.2-0.4=1.0022 t_0.4-0.6=0.8563 t_0.6-0.8=0.7182 t_0.8-1.0=0.6484] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0237 + Deleted old checkpoint: checkpoint_epoch_0234 +[MEM @ epoch 237 end] RAM: 19.8/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 238/299 (79% done) --- + [Epoch 238][10/33] loss=0.924033 avg=0.829272 VRAM=39.4GiB | 79.4% done | ETA(epoch): 637s + [Epoch 238][20/33] loss=0.864241 avg=0.828221 VRAM=39.2GiB | 79.5% done | ETA(epoch): 360s + [Epoch 238][30/33] loss=0.898066 avg=0.833897 VRAM=39.4GiB | 79.6% done | ETA(epoch): 83s + Train loss: 0.836569 (913.3s) ETA: 1023min + Val loss: 0.882360 [t_0.0-0.2=1.0619 t_0.2-0.4=1.0021 t_0.4-0.6=0.8565 t_0.6-0.8=0.7150 t_0.8-1.0=0.6310] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0238 + Deleted old checkpoint: checkpoint_epoch_0235 +[MEM @ epoch 238 end] RAM: 19.9/188.4 GiB (10.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 239/299 (80% done) --- + [Epoch 239][10/33] loss=0.812716 avg=0.864729 VRAM=39.4GiB | 79.8% done | ETA(epoch): 636s + [Epoch 239][20/33] loss=0.812752 avg=0.850594 VRAM=39.2GiB | 79.9% done | ETA(epoch): 360s + [Epoch 239][30/33] loss=0.727791 avg=0.848791 VRAM=39.4GiB | 80.0% done | ETA(epoch): 83s + [MilestoneVis] train_0 step 7919 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 7919 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 7919 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 7919 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 7919 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 7919 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 7919 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 7919 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 7919 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 7919 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + Train loss: 0.849801 (1672.5s) ETA: 1009min + Val loss: 0.848668 [t_0.0-0.2=1.0628 t_0.2-0.4=0.9830 t_0.4-0.6=0.8558 t_0.6-0.8=0.6905 t_0.8-1.0=0.6184] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0239 + Deleted old checkpoint: checkpoint_epoch_0236 +[MEM @ epoch 239 end] RAM: 20.1/188.4 GiB (10.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 240/299 (80% done) --- + [Epoch 240][10/33] loss=0.870498 avg=0.847957 VRAM=39.4GiB | 80.1% done | ETA(epoch): 636s + [Epoch 240][20/33] loss=0.903561 avg=0.857111 VRAM=39.2GiB | 80.2% done | ETA(epoch): 360s + [Epoch 240][30/33] loss=0.853247 avg=0.860680 VRAM=39.4GiB | 80.3% done | ETA(epoch): 83s + Train loss: 0.858335 (913.3s) ETA: 992min + Val loss: 0.846729 [t_0.0-0.2=1.0796 t_0.2-0.4=1.0081 t_0.4-0.6=0.8148 t_0.6-0.8=0.7082 t_0.8-1.0=0.6625] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0240 + Deleted old checkpoint: checkpoint_epoch_0237 +[MEM @ epoch 240 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 241/299 (80% done) --- + [Epoch 241][10/33] loss=0.927830 avg=0.864576 VRAM=39.4GiB | 80.4% done | ETA(epoch): 637s + [Epoch 241][20/33] loss=0.898307 avg=0.867142 VRAM=39.2GiB | 80.5% done | ETA(epoch): 360s + [Epoch 241][30/33] loss=0.784958 avg=0.870157 VRAM=39.4GiB | 80.6% done | ETA(epoch): 83s + Train loss: 0.864167 (913.1s) ETA: 975min + Val loss: 0.850898 [t_0.0-0.2=1.0781 t_0.2-0.4=1.0200 t_0.4-0.6=0.8569 t_0.6-0.8=0.6813 t_0.8-1.0=0.6501] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0241 + Deleted old checkpoint: checkpoint_epoch_0238 +[MEM @ epoch 241 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 242/299 (81% done) --- + [Epoch 242][10/33] loss=0.783926 avg=0.837096 VRAM=39.4GiB | 80.8% done | ETA(epoch): 636s + [Epoch 242][20/33] loss=0.970791 avg=0.842776 VRAM=39.2GiB | 80.9% done | ETA(epoch): 360s + [Epoch 242][30/33] loss=0.728247 avg=0.843497 VRAM=39.4GiB | 81.0% done | ETA(epoch): 83s + Train loss: 0.843021 (912.9s) ETA: 958min + Val loss: 0.832337 [t_0.0-0.2=1.0678 t_0.2-0.4=0.9894 t_0.4-0.6=0.8500 t_0.6-0.8=0.7041 t_0.8-1.0=0.6418] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0242 + Deleted old checkpoint: checkpoint_epoch_0239 +[MEM @ epoch 242 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 243/299 (81% done) --- + [Epoch 243][10/33] loss=0.880877 avg=0.839758 VRAM=39.4GiB | 81.1% done | ETA(epoch): 635s + [Epoch 243][20/33] loss=0.870249 avg=0.870543 VRAM=39.2GiB | 81.2% done | ETA(epoch): 359s + [Epoch 243][30/33] loss=0.903686 avg=0.866763 VRAM=39.4GiB | 81.3% done | ETA(epoch): 83s + Train loss: 0.863849 (912.2s) ETA: 941min + Val loss: 0.884358 [t_0.0-0.2=1.0589 t_0.2-0.4=1.0009 t_0.4-0.6=0.8202 t_0.6-0.8=0.7357 t_0.8-1.0=0.6444] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0243 + Deleted old checkpoint: checkpoint_epoch_0240 +[MEM @ epoch 243 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 244/299 (81% done) --- + [Epoch 244][10/33] loss=0.914559 avg=0.843674 VRAM=39.4GiB | 81.4% done | ETA(epoch): 636s + [Epoch 244][20/33] loss=0.919409 avg=0.850224 VRAM=39.2GiB | 81.5% done | ETA(epoch): 359s + [Epoch 244][30/33] loss=0.712421 avg=0.846908 VRAM=39.4GiB | 81.6% done | ETA(epoch): 83s + Train loss: 0.850579 (912.4s) ETA: 924min + Val loss: 0.862627 [t_0.0-0.2=1.0770 t_0.2-0.4=1.0191 t_0.4-0.6=0.8288 t_0.6-0.8=0.7103 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0244 + Deleted old checkpoint: checkpoint_epoch_0241 +[MEM @ epoch 244 end] RAM: 20.2/188.4 GiB (10.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 245/299 (82% done) --- + [Epoch 245][10/33] loss=0.812668 avg=0.865133 VRAM=39.4GiB | 81.8% done | ETA(epoch): 635s + [Epoch 245][20/33] loss=0.917137 avg=0.870078 VRAM=39.2GiB | 81.9% done | ETA(epoch): 359s + [Epoch 245][30/33] loss=0.816329 avg=0.865876 VRAM=39.4GiB | 82.0% done | ETA(epoch): 83s + Train loss: 0.860150 (912.6s) ETA: 907min + Val loss: 0.836907 [t_0.0-0.2=1.0691 t_0.2-0.4=1.0191 t_0.4-0.6=0.8538 t_0.6-0.8=0.7100 t_0.8-1.0=0.6474] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0245 + Deleted old checkpoint: checkpoint_epoch_0242 +[MEM @ epoch 245 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 246/299 (82% done) --- + [Epoch 246][10/33] loss=0.858128 avg=0.813520 VRAM=39.4GiB | 82.1% done | ETA(epoch): 636s + [Epoch 246][20/33] loss=0.840711 avg=0.815261 VRAM=39.2GiB | 82.2% done | ETA(epoch): 360s + [Epoch 246][30/33] loss=0.762624 avg=0.828382 VRAM=39.4GiB | 82.3% done | ETA(epoch): 83s + Train loss: 0.834651 (913.3s) ETA: 890min + Val loss: 0.873542 [t_0.0-0.2=1.0859 t_0.2-0.4=0.9712 t_0.4-0.6=0.8559 t_0.6-0.8=0.7101 t_0.8-1.0=0.6268] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0246 + Deleted old checkpoint: checkpoint_epoch_0243 +[MEM @ epoch 246 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 247/299 (82% done) --- + [Epoch 247][10/33] loss=0.757409 avg=0.869499 VRAM=39.4GiB | 82.4% done | ETA(epoch): 636s + [Epoch 247][20/33] loss=0.888327 avg=0.864159 VRAM=39.2GiB | 82.5% done | ETA(epoch): 360s + [Epoch 247][30/33] loss=0.852036 avg=0.864714 VRAM=39.4GiB | 82.6% done | ETA(epoch): 83s + Train loss: 0.865672 (913.0s) ETA: 874min + Val loss: 0.836873 [t_0.0-0.2=1.0869 t_0.2-0.4=0.9929 t_0.4-0.6=0.8090 t_0.6-0.8=0.7227 t_0.8-1.0=0.6408] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0247 + Deleted old checkpoint: checkpoint_epoch_0244 +[MEM @ epoch 247 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 248/299 (83% done) --- + [Epoch 248][10/33] loss=0.795078 avg=0.855677 VRAM=39.4GiB | 82.8% done | ETA(epoch): 637s + [Epoch 248][20/33] loss=0.808975 avg=0.847624 VRAM=39.2GiB | 82.9% done | ETA(epoch): 360s + [Epoch 248][30/33] loss=0.774952 avg=0.853098 VRAM=39.4GiB | 83.0% done | ETA(epoch): 83s + Train loss: 0.850180 (913.1s) ETA: 857min + Val loss: 0.855608 [t_0.0-0.2=1.0812 t_0.2-0.4=1.0094 t_0.4-0.6=0.9002 t_0.6-0.8=0.6781 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0248 + Deleted old checkpoint: checkpoint_epoch_0245 +[MEM @ epoch 248 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 249/299 (83% done) --- + [Epoch 249][10/33] loss=0.823864 avg=0.889018 VRAM=39.4GiB | 83.1% done | ETA(epoch): 636s + [Epoch 249][20/33] loss=0.886891 avg=0.890308 VRAM=39.2GiB | 83.2% done | ETA(epoch): 360s + [Epoch 249][30/33] loss=0.843001 avg=0.877468 VRAM=39.4GiB | 83.3% done | ETA(epoch): 83s + Train loss: 0.874784 (913.1s) ETA: 840min + Val loss: 0.857394 [t_0.0-0.2=1.0637 t_0.2-0.4=0.9911 t_0.4-0.6=0.8245 t_0.6-0.8=0.7066 t_0.8-1.0=0.6568] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0249 + Deleted old checkpoint: checkpoint_epoch_0246 +[MEM @ epoch 249 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 250/299 (83% done) --- + [Epoch 250][10/33] loss=0.789994 avg=0.854049 VRAM=39.4GiB | 83.4% done | ETA(epoch): 636s + [Epoch 250][20/33] loss=0.839299 avg=0.827278 VRAM=39.2GiB | 83.5% done | ETA(epoch): 360s + [Epoch 250][30/33] loss=0.907163 avg=0.832830 VRAM=39.4GiB | 83.6% done | ETA(epoch): 83s + Train loss: 0.834257 (912.7s) ETA: 823min + Val loss: 0.816846 [t_0.0-0.2=1.0660 t_0.2-0.4=0.9933 t_0.4-0.6=0.8139 t_0.6-0.8=0.7116 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0250 + Deleted old checkpoint: checkpoint_epoch_0247 +[MEM @ epoch 250 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 251/299 (84% done) --- + [Epoch 251][10/33] loss=0.866107 avg=0.853129 VRAM=39.4GiB | 83.8% done | ETA(epoch): 636s + [Epoch 251][20/33] loss=0.870595 avg=0.856686 VRAM=39.2GiB | 83.9% done | ETA(epoch): 359s + [Epoch 251][30/33] loss=0.864577 avg=0.854922 VRAM=39.4GiB | 84.0% done | ETA(epoch): 83s + Train loss: 0.856888 (913.0s) ETA: 806min + Val loss: 0.851187 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9996 t_0.4-0.6=0.8334 t_0.6-0.8=0.7001 t_0.8-1.0=0.6397] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0251 + Deleted old checkpoint: checkpoint_epoch_0248 +[MEM @ epoch 251 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 252/299 (84% done) --- + [Epoch 252][10/33] loss=0.846758 avg=0.870198 VRAM=39.4GiB | 84.1% done | ETA(epoch): 636s + [Epoch 252][20/33] loss=0.872034 avg=0.854778 VRAM=39.2GiB | 84.2% done | ETA(epoch): 359s + [Epoch 252][30/33] loss=0.876840 avg=0.855628 VRAM=39.4GiB | 84.3% done | ETA(epoch): 83s + Train loss: 0.865414 (912.5s) ETA: 789min + Val loss: 0.841278 [t_0.0-0.2=1.0898 t_0.2-0.4=0.9933 t_0.4-0.6=0.8201 t_0.6-0.8=0.7038 t_0.8-1.0=0.6290] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0252 + Deleted old checkpoint: checkpoint_epoch_0249 +[MEM @ epoch 252 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 253/299 (84% done) --- + [Epoch 253][10/33] loss=0.816250 avg=0.825004 VRAM=39.4GiB | 84.4% done | ETA(epoch): 636s + [Epoch 253][20/33] loss=0.804101 avg=0.832300 VRAM=39.2GiB | 84.5% done | ETA(epoch): 359s + [Epoch 253][30/33] loss=0.871264 avg=0.839902 VRAM=39.4GiB | 84.6% done | ETA(epoch): 83s + Train loss: 0.840725 (912.7s) ETA: 772min + Val loss: 0.880940 [t_0.0-0.2=1.0680 t_0.2-0.4=1.0082 t_0.4-0.6=0.8689 t_0.6-0.8=0.7162 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0253 + Deleted old checkpoint: checkpoint_epoch_0250 +[MEM @ epoch 253 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 254/299 (85% done) --- + [Epoch 254][10/33] loss=0.862797 avg=0.846840 VRAM=39.4GiB | 84.8% done | ETA(epoch): 636s + [Epoch 254][20/33] loss=0.792928 avg=0.842700 VRAM=39.2GiB | 84.9% done | ETA(epoch): 360s + [Epoch 254][30/33] loss=0.810293 avg=0.836429 VRAM=39.4GiB | 85.0% done | ETA(epoch): 83s + Train loss: 0.840481 (912.8s) ETA: 755min + Val loss: 0.882845 [t_0.0-0.2=1.0639 t_0.2-0.4=1.0009 t_0.4-0.6=0.8770 t_0.6-0.8=0.7243 t_0.8-1.0=0.6302] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0254 + Deleted old checkpoint: checkpoint_epoch_0251 +[MEM @ epoch 254 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 255/299 (85% done) --- + [Epoch 255][10/33] loss=0.893532 avg=0.856446 VRAM=39.4GiB | 85.1% done | ETA(epoch): 636s + [Epoch 255][20/33] loss=0.789192 avg=0.863305 VRAM=39.2GiB | 85.2% done | ETA(epoch): 360s + [Epoch 255][30/33] loss=0.906499 avg=0.869101 VRAM=39.4GiB | 85.3% done | ETA(epoch): 83s + Train loss: 0.868798 (912.8s) ETA: 738min + Val loss: 0.860283 [t_0.0-0.2=1.0809 t_0.2-0.4=1.0131 t_0.4-0.6=0.8093 t_0.6-0.8=0.6923 t_0.8-1.0=0.6443] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0255 + Deleted old checkpoint: checkpoint_epoch_0252 +[MEM @ epoch 255 end] RAM: 20.3/188.4 GiB (10.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 256/299 (85% done) --- + [Epoch 256][10/33] loss=0.872559 avg=0.875403 VRAM=39.4GiB | 85.4% done | ETA(epoch): 636s + [Epoch 256][20/33] loss=0.880033 avg=0.865364 VRAM=39.2GiB | 85.5% done | ETA(epoch): 360s + [Epoch 256][30/33] loss=0.879419 avg=0.862606 VRAM=39.4GiB | 85.6% done | ETA(epoch): 83s + Train loss: 0.861886 (913.6s) ETA: 722min + Val loss: 0.841265 [t_0.0-0.2=1.0734 t_0.2-0.4=1.0177 t_0.4-0.6=0.8527 t_0.6-0.8=0.7081 t_0.8-1.0=0.6287] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0256 + Deleted old checkpoint: checkpoint_epoch_0253 +[MEM @ epoch 256 end] RAM: 21.7/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 257/299 (86% done) --- + [Epoch 257][10/33] loss=0.813759 avg=0.858439 VRAM=39.4GiB | 85.8% done | ETA(epoch): 636s + [Epoch 257][20/33] loss=0.974069 avg=0.861485 VRAM=39.2GiB | 85.9% done | ETA(epoch): 360s + [Epoch 257][30/33] loss=0.852902 avg=0.866524 VRAM=39.4GiB | 86.0% done | ETA(epoch): 83s + Train loss: 0.862819 (912.9s) ETA: 705min + Val loss: 0.859419 [t_0.0-0.2=1.0708 t_0.2-0.4=0.9554 t_0.4-0.6=0.8217 t_0.6-0.8=0.7055 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0257 + Deleted old checkpoint: checkpoint_epoch_0254 +[MEM @ epoch 257 end] RAM: 21.6/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 258/299 (86% done) --- + [Epoch 258][10/33] loss=0.960379 avg=0.896877 VRAM=39.4GiB | 86.1% done | ETA(epoch): 636s + [Epoch 258][20/33] loss=0.864482 avg=0.884051 VRAM=39.2GiB | 86.2% done | ETA(epoch): 359s + [Epoch 258][30/33] loss=0.883338 avg=0.876798 VRAM=39.4GiB | 86.3% done | ETA(epoch): 83s + Train loss: 0.876638 (912.6s) ETA: 688min + Val loss: 0.875292 [t_0.0-0.2=1.0740 t_0.2-0.4=1.0010 t_0.4-0.6=0.8448 t_0.6-0.8=0.7018 t_0.8-1.0=0.6366] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0258 + Deleted old checkpoint: checkpoint_epoch_0255 +[MEM @ epoch 258 end] RAM: 21.4/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 259/299 (86% done) --- + [Epoch 259][10/33] loss=0.833364 avg=0.864899 VRAM=39.4GiB | 86.4% done | ETA(epoch): 637s + [Epoch 259][20/33] loss=0.902626 avg=0.868921 VRAM=39.2GiB | 86.5% done | ETA(epoch): 360s + [Epoch 259][30/33] loss=0.786486 avg=0.861115 VRAM=39.4GiB | 86.6% done | ETA(epoch): 83s + Train loss: 0.860587 (913.6s) ETA: 671min + Val loss: 0.870524 [t_0.0-0.2=1.0764 t_0.2-0.4=0.9811 t_0.4-0.6=0.8270 t_0.6-0.8=0.7131 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0259 + Deleted old checkpoint: checkpoint_epoch_0256 +[MEM @ epoch 259 end] RAM: 21.6/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 260/299 (87% done) --- + [Epoch 260][10/33] loss=0.890540 avg=0.855209 VRAM=39.4GiB | 86.8% done | ETA(epoch): 637s + [Epoch 260][20/33] loss=0.890873 avg=0.843320 VRAM=39.2GiB | 86.9% done | ETA(epoch): 360s + [Epoch 260][30/33] loss=0.842694 avg=0.850709 VRAM=39.4GiB | 87.0% done | ETA(epoch): 83s + Train loss: 0.851041 (913.1s) ETA: 654min + Val loss: 0.835154 [t_0.0-0.2=1.0681 t_0.2-0.4=0.9463 t_0.4-0.6=0.8766 t_0.6-0.8=0.6974 t_0.8-1.0=0.6219] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0260 + Deleted old checkpoint: checkpoint_epoch_0257 +[MEM @ epoch 260 end] RAM: 21.5/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 261/299 (87% done) --- + [Epoch 261][10/33] loss=0.939947 avg=0.879821 VRAM=39.4GiB | 87.1% done | ETA(epoch): 636s + [Epoch 261][20/33] loss=0.907913 avg=0.858847 VRAM=39.2GiB | 87.2% done | ETA(epoch): 360s + [Epoch 261][30/33] loss=0.819618 avg=0.861138 VRAM=39.4GiB | 87.3% done | ETA(epoch): 83s + Train loss: 0.856960 (913.0s) ETA: 637min + Val loss: 0.847273 [t_0.0-0.2=1.0729 t_0.2-0.4=1.0042 t_0.4-0.6=0.8561 t_0.6-0.8=0.6716 t_0.8-1.0=0.6264] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0261 + Deleted old checkpoint: checkpoint_epoch_0258 +[MEM @ epoch 261 end] RAM: 21.6/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 262/299 (87% done) --- + [Epoch 262][10/33] loss=0.835615 avg=0.838261 VRAM=39.4GiB | 87.4% done | ETA(epoch): 636s + [Epoch 262][20/33] loss=0.944760 avg=0.848713 VRAM=39.2GiB | 87.5% done | ETA(epoch): 360s + [Epoch 262][30/33] loss=0.800903 avg=0.848741 VRAM=39.4GiB | 87.6% done | ETA(epoch): 83s + Train loss: 0.853630 (912.9s) ETA: 621min + Val loss: 0.851750 [t_0.0-0.2=1.0728 t_0.2-0.4=0.9885 t_0.4-0.6=0.8694 t_0.6-0.8=0.7166 t_0.8-1.0=0.6545] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0262 + Deleted old checkpoint: checkpoint_epoch_0259 +[MEM @ epoch 262 end] RAM: 21.6/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 263/299 (88% done) --- + [Epoch 263][10/33] loss=0.886477 avg=0.855773 VRAM=39.4GiB | 87.8% done | ETA(epoch): 636s + [Epoch 263][20/33] loss=0.831144 avg=0.858343 VRAM=39.2GiB | 87.9% done | ETA(epoch): 360s + [Epoch 263][30/33] loss=0.759328 avg=0.856238 VRAM=39.4GiB | 88.0% done | ETA(epoch): 83s + Train loss: 0.857762 (913.1s) ETA: 604min + Val loss: 0.891653 [t_0.0-0.2=1.0733 t_0.2-0.4=0.9980 t_0.4-0.6=0.8513 t_0.6-0.8=0.7328 t_0.8-1.0=0.6138] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0263 + Deleted old checkpoint: checkpoint_epoch_0260 +[MEM @ epoch 263 end] RAM: 21.5/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 264/299 (88% done) --- + [Epoch 264][10/33] loss=0.908377 avg=0.874073 VRAM=39.4GiB | 88.1% done | ETA(epoch): 637s + [Epoch 264][20/33] loss=0.862854 avg=0.853917 VRAM=39.2GiB | 88.2% done | ETA(epoch): 360s + [Epoch 264][30/33] loss=0.874056 avg=0.857918 VRAM=39.4GiB | 88.3% done | ETA(epoch): 83s + Train loss: 0.855367 (913.6s) ETA: 587min + Val loss: 0.855561 [t_0.0-0.2=1.0644 t_0.2-0.4=1.0104 t_0.4-0.6=0.8637 t_0.6-0.8=0.6960 t_0.8-1.0=0.6159] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0264 + Deleted old checkpoint: checkpoint_epoch_0261 +[MEM @ epoch 264 end] RAM: 21.6/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 265/299 (88% done) --- + [Epoch 265][10/33] loss=0.826781 avg=0.841663 VRAM=39.4GiB | 88.4% done | ETA(epoch): 636s + [Epoch 265][20/33] loss=0.959930 avg=0.866602 VRAM=39.2GiB | 88.5% done | ETA(epoch): 360s + [Epoch 265][30/33] loss=0.854334 avg=0.857502 VRAM=39.4GiB | 88.6% done | ETA(epoch): 83s + Train loss: 0.860273 (913.1s) ETA: 570min + Val loss: 0.888357 [t_0.0-0.2=1.0832 t_0.2-0.4=0.9951 t_0.4-0.6=0.8586 t_0.6-0.8=0.7167 t_0.8-1.0=0.6404] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0265 + Deleted old checkpoint: checkpoint_epoch_0262 +[MEM @ epoch 265 end] RAM: 21.6/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 266/299 (89% done) --- + [Epoch 266][10/33] loss=0.838069 avg=0.864365 VRAM=39.4GiB | 88.8% done | ETA(epoch): 636s + [Epoch 266][20/33] loss=0.795160 avg=0.855077 VRAM=39.2GiB | 88.9% done | ETA(epoch): 360s + [Epoch 266][30/33] loss=0.839936 avg=0.855470 VRAM=39.4GiB | 89.0% done | ETA(epoch): 83s + Train loss: 0.851051 (913.4s) ETA: 553min + Val loss: 0.840643 [t_0.0-0.2=1.0771 t_0.2-0.4=1.0004 t_0.4-0.6=0.8469 t_0.6-0.8=0.7088 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0266 + Deleted old checkpoint: checkpoint_epoch_0263 +[MEM @ epoch 266 end] RAM: 21.5/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 267/299 (89% done) --- + [Epoch 267][10/33] loss=0.808410 avg=0.848100 VRAM=39.4GiB | 89.1% done | ETA(epoch): 636s + [Epoch 267][20/33] loss=0.816854 avg=0.844197 VRAM=39.2GiB | 89.2% done | ETA(epoch): 360s + [Epoch 267][30/33] loss=0.779598 avg=0.851482 VRAM=39.4GiB | 89.3% done | ETA(epoch): 83s + Train loss: 0.856702 (913.1s) ETA: 536min + Val loss: 0.874148 [t_0.0-0.2=1.0823 t_0.2-0.4=1.0172 t_0.4-0.6=0.8037 t_0.6-0.8=0.7325 t_0.8-1.0=0.6644] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0267 + Deleted old checkpoint: checkpoint_epoch_0264 +[MEM @ epoch 267 end] RAM: 21.5/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 268/299 (89% done) --- + [Epoch 268][10/33] loss=0.907343 avg=0.854003 VRAM=39.4GiB | 89.4% done | ETA(epoch): 636s + [Epoch 268][20/33] loss=0.882347 avg=0.841611 VRAM=39.2GiB | 89.5% done | ETA(epoch): 359s + [Epoch 268][30/33] loss=0.802130 avg=0.838122 VRAM=39.4GiB | 89.6% done | ETA(epoch): 83s + Train loss: 0.840265 (912.8s) ETA: 520min + Val loss: 0.886057 [t_0.0-0.2=1.0646 t_0.2-0.4=1.0090 t_0.4-0.6=0.8530 t_0.6-0.8=0.7154 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0268 + Deleted old checkpoint: checkpoint_epoch_0265 +[MEM @ epoch 268 end] RAM: 21.6/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 269/299 (90% done) --- + [Epoch 269][10/33] loss=0.730992 avg=0.825194 VRAM=39.4GiB | 89.8% done | ETA(epoch): 637s + [Epoch 269][20/33] loss=0.798214 avg=0.830085 VRAM=39.2GiB | 89.9% done | ETA(epoch): 360s + [Epoch 269][30/33] loss=0.861976 avg=0.840781 VRAM=39.4GiB | 90.0% done | ETA(epoch): 83s + [MilestoneVis] train_0 step 8909 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 8909 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 8909 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 8909 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 8909 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 8909 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 8909 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 8909 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 8909 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 8909 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + Train loss: 0.846281 (1673.6s) ETA: 504min + Val loss: 0.847681 [t_0.0-0.2=1.0906 t_0.2-0.4=0.9691 t_0.4-0.6=0.8387 t_0.6-0.8=0.7185 t_0.8-1.0=0.6524] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0269 + Deleted old checkpoint: checkpoint_epoch_0266 +[MEM @ epoch 269 end] RAM: 22.1/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 270/299 (90% done) --- + [Epoch 270][10/33] loss=0.855872 avg=0.857920 VRAM=39.4GiB | 90.1% done | ETA(epoch): 636s + [Epoch 270][20/33] loss=0.795260 avg=0.845376 VRAM=39.2GiB | 90.2% done | ETA(epoch): 359s + [Epoch 270][30/33] loss=0.882479 avg=0.844609 VRAM=39.4GiB | 90.3% done | ETA(epoch): 83s + Train loss: 0.845284 (912.7s) ETA: 487min + Val loss: 0.851773 [t_0.0-0.2=1.0548 t_0.2-0.4=0.9704 t_0.4-0.6=0.8496 t_0.6-0.8=0.7177 t_0.8-1.0=0.6293] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0270 + Deleted old checkpoint: checkpoint_epoch_0267 +[MEM @ epoch 270 end] RAM: 22.1/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 271/299 (90% done) --- + [Epoch 271][10/33] loss=0.886514 avg=0.864283 VRAM=39.4GiB | 90.4% done | ETA(epoch): 636s + [Epoch 271][20/33] loss=0.840639 avg=0.866288 VRAM=39.2GiB | 90.5% done | ETA(epoch): 360s + [Epoch 271][30/33] loss=0.954924 avg=0.859693 VRAM=39.4GiB | 90.6% done | ETA(epoch): 83s + Train loss: 0.858106 (913.3s) ETA: 470min + Val loss: 0.869742 [t_0.0-0.2=1.0591 t_0.2-0.4=1.0122 t_0.4-0.6=0.8372 t_0.6-0.8=0.7252 t_0.8-1.0=0.6366] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0271 + Deleted old checkpoint: checkpoint_epoch_0268 +[MEM @ epoch 271 end] RAM: 22.3/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 272/299 (91% done) --- + [Epoch 272][10/33] loss=0.836990 avg=0.839835 VRAM=39.4GiB | 90.8% done | ETA(epoch): 637s + [Epoch 272][20/33] loss=0.842281 avg=0.844107 VRAM=39.2GiB | 90.9% done | ETA(epoch): 360s + [Epoch 272][30/33] loss=0.886080 avg=0.849363 VRAM=39.4GiB | 91.0% done | ETA(epoch): 83s + Train loss: 0.846444 (913.2s) ETA: 454min + Val loss: 0.860459 [t_0.0-0.2=1.0787 t_0.2-0.4=1.0099 t_0.4-0.6=0.8474 t_0.6-0.8=0.7105 t_0.8-1.0=0.6605] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0272 + Deleted old checkpoint: checkpoint_epoch_0269 +[MEM @ epoch 272 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 273/299 (91% done) --- + [Epoch 273][10/33] loss=0.749059 avg=0.840199 VRAM=39.4GiB | 91.1% done | ETA(epoch): 636s + [Epoch 273][20/33] loss=0.803302 avg=0.856085 VRAM=39.2GiB | 91.2% done | ETA(epoch): 360s + [Epoch 273][30/33] loss=0.851905 avg=0.861375 VRAM=39.4GiB | 91.3% done | ETA(epoch): 83s + Train loss: 0.860919 (913.6s) ETA: 437min + Val loss: 0.815714 [t_0.0-0.2=1.0960 t_0.2-0.4=1.0044 t_0.4-0.6=0.8183 t_0.6-0.8=0.6923 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0273 + Deleted old checkpoint: checkpoint_epoch_0270 +[MEM @ epoch 273 end] RAM: 22.1/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 274/299 (91% done) --- + [Epoch 274][10/33] loss=0.881112 avg=0.876544 VRAM=39.4GiB | 91.4% done | ETA(epoch): 637s + [Epoch 274][20/33] loss=0.847269 avg=0.865843 VRAM=39.2GiB | 91.5% done | ETA(epoch): 360s + [Epoch 274][30/33] loss=0.779153 avg=0.853292 VRAM=39.4GiB | 91.6% done | ETA(epoch): 83s + Train loss: 0.856554 (913.8s) ETA: 420min + Val loss: 0.852373 [t_0.0-0.2=1.0648 t_0.2-0.4=0.9759 t_0.4-0.6=0.8437 t_0.6-0.8=0.7110 t_0.8-1.0=0.6306] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0274 + Deleted old checkpoint: checkpoint_epoch_0271 +[MEM @ epoch 274 end] RAM: 22.3/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 275/299 (92% done) --- + [Epoch 275][10/33] loss=0.862040 avg=0.844158 VRAM=39.4GiB | 91.8% done | ETA(epoch): 637s + [Epoch 275][20/33] loss=0.942064 avg=0.845397 VRAM=39.2GiB | 91.9% done | ETA(epoch): 360s + [Epoch 275][30/33] loss=0.825191 avg=0.852097 VRAM=39.4GiB | 92.0% done | ETA(epoch): 83s + Train loss: 0.850337 (913.5s) ETA: 403min + Val loss: 0.850326 [t_0.0-0.2=1.0713 t_0.2-0.4=0.9975 t_0.4-0.6=0.8425 t_0.6-0.8=0.7068 t_0.8-1.0=0.6498] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0275 + Deleted old checkpoint: checkpoint_epoch_0272 +[MEM @ epoch 275 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 276/299 (92% done) --- + [Epoch 276][10/33] loss=0.869948 avg=0.844377 VRAM=39.4GiB | 92.1% done | ETA(epoch): 636s + [Epoch 276][20/33] loss=0.809686 avg=0.846597 VRAM=39.2GiB | 92.2% done | ETA(epoch): 360s + [Epoch 276][30/33] loss=0.876100 avg=0.852017 VRAM=39.4GiB | 92.3% done | ETA(epoch): 83s + Train loss: 0.852290 (913.6s) ETA: 386min + Val loss: 0.858253 [t_0.0-0.2=1.0678 t_0.2-0.4=1.0325 t_0.4-0.6=0.8365 t_0.6-0.8=0.6885 t_0.8-1.0=0.6352] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0276 + Deleted old checkpoint: checkpoint_epoch_0273 +[MEM @ epoch 276 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 277/299 (92% done) --- + [Epoch 277][10/33] loss=0.879320 avg=0.858658 VRAM=39.4GiB | 92.4% done | ETA(epoch): 637s + [Epoch 277][20/33] loss=0.807437 avg=0.842979 VRAM=39.2GiB | 92.5% done | ETA(epoch): 360s + [Epoch 277][30/33] loss=0.888125 avg=0.842250 VRAM=39.4GiB | 92.6% done | ETA(epoch): 83s + Train loss: 0.841603 (914.8s) ETA: 369min + Val loss: 0.829573 [t_0.0-0.2=1.0727 t_0.2-0.4=0.9806 t_0.4-0.6=0.8525 t_0.6-0.8=0.7055 t_0.8-1.0=0.6312] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0277 + Deleted old checkpoint: checkpoint_epoch_0274 +[MEM @ epoch 277 end] RAM: 22.0/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 278/299 (93% done) --- + [Epoch 278][10/33] loss=0.989428 avg=0.866807 VRAM=39.4GiB | 92.8% done | ETA(epoch): 637s + [Epoch 278][20/33] loss=0.832683 avg=0.850553 VRAM=39.2GiB | 92.9% done | ETA(epoch): 360s + [Epoch 278][30/33] loss=0.814678 avg=0.858275 VRAM=39.4GiB | 93.0% done | ETA(epoch): 83s + Train loss: 0.858438 (914.4s) ETA: 353min + Val loss: 0.858022 [t_0.0-0.2=1.0839 t_0.2-0.4=1.0000 t_0.4-0.6=0.8471 t_0.6-0.8=0.6961 t_0.8-1.0=0.6402] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0278 + Deleted old checkpoint: checkpoint_epoch_0275 +[MEM @ epoch 278 end] RAM: 22.3/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 279/299 (93% done) --- + [Epoch 279][10/33] loss=0.909187 avg=0.866417 VRAM=39.4GiB | 93.1% done | ETA(epoch): 637s + [Epoch 279][20/33] loss=0.826248 avg=0.858016 VRAM=39.2GiB | 93.2% done | ETA(epoch): 360s + [Epoch 279][30/33] loss=0.778924 avg=0.852454 VRAM=39.4GiB | 93.3% done | ETA(epoch): 83s + Train loss: 0.850377 (914.1s) ETA: 336min + Val loss: 0.910099 [t_0.0-0.2=1.0725 t_0.2-0.4=1.0043 t_0.4-0.6=0.8499 t_0.6-0.8=0.6860 t_0.8-1.0=0.6358] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0279 + Deleted old checkpoint: checkpoint_epoch_0276 +[MEM @ epoch 279 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 280/299 (93% done) --- + [Epoch 280][10/33] loss=0.860865 avg=0.860760 VRAM=39.4GiB | 93.4% done | ETA(epoch): 637s + [Epoch 280][20/33] loss=0.850525 avg=0.857492 VRAM=39.2GiB | 93.5% done | ETA(epoch): 360s + [Epoch 280][30/33] loss=0.868195 avg=0.860349 VRAM=39.4GiB | 93.6% done | ETA(epoch): 83s + Train loss: 0.855233 (913.8s) ETA: 319min + Val loss: 0.848244 [t_0.0-0.2=1.0570 t_0.2-0.4=1.0078 t_0.4-0.6=0.8032 t_0.6-0.8=0.7489 t_0.8-1.0=0.6324] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0280 + Deleted old checkpoint: checkpoint_epoch_0277 +[MEM @ epoch 280 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 281/299 (94% done) --- + [Epoch 281][10/33] loss=0.774643 avg=0.852573 VRAM=39.4GiB | 93.8% done | ETA(epoch): 637s + [Epoch 281][20/33] loss=0.897272 avg=0.869196 VRAM=39.2GiB | 93.9% done | ETA(epoch): 360s + [Epoch 281][30/33] loss=0.881880 avg=0.859512 VRAM=39.4GiB | 94.0% done | ETA(epoch): 83s + Train loss: 0.857619 (913.3s) ETA: 302min + Val loss: 0.841680 [t_0.0-0.2=1.0774 t_0.2-0.4=1.0001 t_0.4-0.6=0.8561 t_0.6-0.8=0.7261 t_0.8-1.0=0.6415] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0281 + Deleted old checkpoint: checkpoint_epoch_0278 +[MEM @ epoch 281 end] RAM: 22.1/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 282/299 (94% done) --- + [Epoch 282][10/33] loss=0.864845 avg=0.869399 VRAM=39.4GiB | 94.1% done | ETA(epoch): 637s + [Epoch 282][20/33] loss=0.871691 avg=0.867362 VRAM=39.2GiB | 94.2% done | ETA(epoch): 360s + [Epoch 282][30/33] loss=0.833683 avg=0.872562 VRAM=39.4GiB | 94.3% done | ETA(epoch): 83s + Train loss: 0.869176 (912.9s) ETA: 285min + Val loss: 0.836712 [t_0.0-0.2=1.0706 t_0.2-0.4=1.0101 t_0.4-0.6=0.8513 t_0.6-0.8=0.7022 t_0.8-1.0=0.6379] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0282 + Deleted old checkpoint: checkpoint_epoch_0279 +[MEM @ epoch 282 end] RAM: 22.0/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 283/299 (94% done) --- + [Epoch 283][10/33] loss=0.738724 avg=0.815877 VRAM=39.4GiB | 94.4% done | ETA(epoch): 636s + [Epoch 283][20/33] loss=0.799959 avg=0.829769 VRAM=39.2GiB | 94.5% done | ETA(epoch): 360s + [Epoch 283][30/33] loss=0.806415 avg=0.825768 VRAM=39.4GiB | 94.6% done | ETA(epoch): 83s + Train loss: 0.823767 (912.8s) ETA: 269min + Val loss: 0.853685 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0224 t_0.4-0.6=0.8475 t_0.6-0.8=0.7125 t_0.8-1.0=0.6268] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0283 + Deleted old checkpoint: checkpoint_epoch_0280 +[MEM @ epoch 283 end] RAM: 22.0/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 284/299 (95% done) --- + [Epoch 284][10/33] loss=0.861785 avg=0.844548 VRAM=39.4GiB | 94.8% done | ETA(epoch): 636s + [Epoch 284][20/33] loss=0.988537 avg=0.860382 VRAM=39.2GiB | 94.9% done | ETA(epoch): 360s + [Epoch 284][30/33] loss=0.925245 avg=0.859496 VRAM=39.4GiB | 95.0% done | ETA(epoch): 83s + Train loss: 0.859150 (912.5s) ETA: 252min + Val loss: 0.865838 [t_0.0-0.2=1.0852 t_0.2-0.4=1.0185 t_0.4-0.6=0.7971 t_0.6-0.8=0.7099 t_0.8-1.0=0.6165] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0284 + Deleted old checkpoint: checkpoint_epoch_0281 +[MEM @ epoch 284 end] RAM: 22.1/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 285/299 (95% done) --- + [Epoch 285][10/33] loss=0.928628 avg=0.866119 VRAM=39.4GiB | 95.1% done | ETA(epoch): 636s + [Epoch 285][20/33] loss=0.865571 avg=0.859153 VRAM=39.2GiB | 95.2% done | ETA(epoch): 359s + [Epoch 285][30/33] loss=0.781204 avg=0.857086 VRAM=39.4GiB | 95.3% done | ETA(epoch): 83s + Train loss: 0.855137 (912.7s) ETA: 235min + Val loss: 0.894450 [t_0.0-0.2=1.0738 t_0.2-0.4=0.9872 t_0.4-0.6=0.8488 t_0.6-0.8=0.7224 t_0.8-1.0=0.6065] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0285 + Deleted old checkpoint: checkpoint_epoch_0282 +[MEM @ epoch 285 end] RAM: 22.1/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 286/299 (95% done) --- + [Epoch 286][10/33] loss=0.852317 avg=0.849756 VRAM=39.4GiB | 95.4% done | ETA(epoch): 637s + [Epoch 286][20/33] loss=0.809586 avg=0.856376 VRAM=39.2GiB | 95.5% done | ETA(epoch): 360s + [Epoch 286][30/33] loss=0.782696 avg=0.858638 VRAM=39.4GiB | 95.6% done | ETA(epoch): 83s + Train loss: 0.855764 (914.2s) ETA: 218min + Val loss: 0.895184 [t_0.0-0.2=1.0675 t_0.2-0.4=0.9945 t_0.4-0.6=0.8394 t_0.6-0.8=0.7072 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0286 + Deleted old checkpoint: checkpoint_epoch_0283 +[MEM @ epoch 286 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 287/299 (96% done) --- + [Epoch 287][10/33] loss=0.880008 avg=0.857485 VRAM=39.4GiB | 95.8% done | ETA(epoch): 636s + [Epoch 287][20/33] loss=0.891493 avg=0.864603 VRAM=39.2GiB | 95.9% done | ETA(epoch): 360s + [Epoch 287][30/33] loss=0.757789 avg=0.851646 VRAM=39.4GiB | 96.0% done | ETA(epoch): 83s + Train loss: 0.849441 (913.2s) ETA: 201min + Val loss: 0.870399 [t_0.0-0.2=1.0907 t_0.2-0.4=0.9707 t_0.4-0.6=0.8592 t_0.6-0.8=0.7138 t_0.8-1.0=0.6511] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0287 + Deleted old checkpoint: checkpoint_epoch_0284 +[MEM @ epoch 287 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 288/299 (96% done) --- + [Epoch 288][10/33] loss=0.815184 avg=0.841935 VRAM=39.4GiB | 96.1% done | ETA(epoch): 636s + [Epoch 288][20/33] loss=0.859792 avg=0.848821 VRAM=39.2GiB | 96.2% done | ETA(epoch): 360s + [Epoch 288][30/33] loss=0.796481 avg=0.842832 VRAM=39.4GiB | 96.3% done | ETA(epoch): 83s + Train loss: 0.845353 (913.2s) ETA: 185min + Val loss: 0.832834 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0002 t_0.4-0.6=0.8460 t_0.6-0.8=0.7070 t_0.8-1.0=0.6286] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0288 + Deleted old checkpoint: checkpoint_epoch_0285 +[MEM @ epoch 288 end] RAM: 22.1/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 289/299 (96% done) --- + [Epoch 289][10/33] loss=0.931831 avg=0.868056 VRAM=39.4GiB | 96.4% done | ETA(epoch): 637s + [Epoch 289][20/33] loss=0.893250 avg=0.864428 VRAM=39.2GiB | 96.5% done | ETA(epoch): 360s + [Epoch 289][30/33] loss=0.855677 avg=0.854562 VRAM=39.4GiB | 96.6% done | ETA(epoch): 83s + Train loss: 0.855931 (913.2s) ETA: 168min + Val loss: 0.859991 [t_0.0-0.2=1.0840 t_0.2-0.4=0.9540 t_0.4-0.6=0.8274 t_0.6-0.8=0.7012 t_0.8-1.0=0.6155] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0289 + Deleted old checkpoint: checkpoint_epoch_0286 +[MEM @ epoch 289 end] RAM: 22.0/188.4 GiB (11.7%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 290/299 (97% done) --- + [Epoch 290][10/33] loss=0.892477 avg=0.847665 VRAM=39.4GiB | 96.8% done | ETA(epoch): 636s + [Epoch 290][20/33] loss=0.831302 avg=0.860025 VRAM=39.2GiB | 96.9% done | ETA(epoch): 359s + [Epoch 290][30/33] loss=0.869616 avg=0.865047 VRAM=39.4GiB | 97.0% done | ETA(epoch): 83s + Train loss: 0.867377 (912.5s) ETA: 151min + Val loss: 0.873727 [t_0.0-0.2=1.0822 t_0.2-0.4=1.0126 t_0.4-0.6=0.8433 t_0.6-0.8=0.7274 t_0.8-1.0=0.6431] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0290 + Deleted old checkpoint: checkpoint_epoch_0287 +[MEM @ epoch 290 end] RAM: 22.2/188.4 GiB (11.8%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 291/299 (97% done) --- + [Epoch 291][10/33] loss=0.917674 avg=0.842512 VRAM=39.4GiB | 97.1% done | ETA(epoch): 636s + [Epoch 291][20/33] loss=0.876790 avg=0.847872 VRAM=39.2GiB | 97.2% done | ETA(epoch): 360s + [Epoch 291][30/33] loss=0.894103 avg=0.850815 VRAM=39.4GiB | 97.3% done | ETA(epoch): 83s + Train loss: 0.854616 (912.4s) ETA: 134min + Val loss: 0.837804 [t_0.0-0.2=1.0711 t_0.2-0.4=0.9490 t_0.4-0.6=0.8349 t_0.6-0.8=0.7223 t_0.8-1.0=0.6479] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0291 + Deleted old checkpoint: checkpoint_epoch_0288 +[MEM @ epoch 291 end] RAM: 22.7/188.4 GiB (12.0%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 292/299 (97% done) --- + [Epoch 292][10/33] loss=0.922054 avg=0.895228 VRAM=39.4GiB | 97.4% done | ETA(epoch): 635s + [Epoch 292][20/33] loss=0.809075 avg=0.882063 VRAM=39.2GiB | 97.5% done | ETA(epoch): 359s + [Epoch 292][30/33] loss=0.824346 avg=0.868531 VRAM=39.4GiB | 97.6% done | ETA(epoch): 83s + Train loss: 0.863800 (911.9s) ETA: 117min + Val loss: 0.877125 [t_0.0-0.2=1.0652 t_0.2-0.4=0.9770 t_0.4-0.6=0.8452 t_0.6-0.8=0.7248 t_0.8-1.0=0.6300] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0292 + Deleted old checkpoint: checkpoint_epoch_0289 +[MEM @ epoch 292 end] RAM: 21.8/188.4 GiB (11.6%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 293/299 (98% done) --- + [Epoch 293][10/33] loss=0.744998 avg=0.861490 VRAM=39.4GiB | 97.8% done | ETA(epoch): 635s + [Epoch 293][20/33] loss=0.906057 avg=0.863321 VRAM=39.2GiB | 97.9% done | ETA(epoch): 359s + [Epoch 293][30/33] loss=0.837411 avg=0.857154 VRAM=39.4GiB | 98.0% done | ETA(epoch): 83s + Train loss: 0.855620 (912.1s) ETA: 101min + Val loss: 0.848718 [t_0.0-0.2=1.0680 t_0.2-0.4=1.0022 t_0.4-0.6=0.8384 t_0.6-0.8=0.7153 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0293 + Deleted old checkpoint: checkpoint_epoch_0290 +[MEM @ epoch 293 end] RAM: 21.6/188.4 GiB (11.5%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 294/299 (98% done) --- + [Epoch 294][10/33] loss=0.785604 avg=0.848753 VRAM=39.4GiB | 98.1% done | ETA(epoch): 636s + [Epoch 294][20/33] loss=0.972560 avg=0.863470 VRAM=39.2GiB | 98.2% done | ETA(epoch): 359s + [Epoch 294][30/33] loss=0.782602 avg=0.862525 VRAM=39.4GiB | 98.3% done | ETA(epoch): 83s + Train loss: 0.862987 (912.5s) ETA: 84min + Val loss: 0.832805 [t_0.0-0.2=1.0734 t_0.2-0.4=1.0058 t_0.4-0.6=0.8416 t_0.6-0.8=0.7116 t_0.8-1.0=0.6439] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0294 + Deleted old checkpoint: checkpoint_epoch_0291 +[MEM @ epoch 294 end] RAM: 21.3/188.4 GiB (11.3%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 295/299 (98% done) --- + [Epoch 295][10/33] loss=0.931806 avg=0.881128 VRAM=39.4GiB | 98.4% done | ETA(epoch): 636s + [Epoch 295][20/33] loss=0.843934 avg=0.878405 VRAM=39.2GiB | 98.5% done | ETA(epoch): 360s + [Epoch 295][30/33] loss=0.961346 avg=0.874104 VRAM=39.4GiB | 98.6% done | ETA(epoch): 83s + Train loss: 0.876785 (912.8s) ETA: 67min + Val loss: 0.860951 [t_0.0-0.2=1.0735 t_0.2-0.4=1.0196 t_0.4-0.6=0.8567 t_0.6-0.8=0.7150 t_0.8-1.0=0.6358] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0295 + Deleted old checkpoint: checkpoint_epoch_0292 +[MEM @ epoch 295 end] RAM: 21.5/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 296/299 (99% done) --- + [Epoch 296][10/33] loss=0.898771 avg=0.857468 VRAM=39.4GiB | 98.8% done | ETA(epoch): 636s + [Epoch 296][20/33] loss=0.833157 avg=0.842033 VRAM=39.2GiB | 98.9% done | ETA(epoch): 360s + [Epoch 296][30/33] loss=0.779545 avg=0.844223 VRAM=39.4GiB | 99.0% done | ETA(epoch): 83s + Train loss: 0.845152 (912.8s) ETA: 50min + Val loss: 0.865256 [t_0.0-0.2=1.0657 t_0.2-0.4=1.0261 t_0.4-0.6=0.8276 t_0.6-0.8=0.7241 t_0.8-1.0=0.6258] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0296 + Deleted old checkpoint: checkpoint_epoch_0293 +[MEM @ epoch 296 end] RAM: 21.5/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 297/299 (99% done) --- + [Epoch 297][10/33] loss=0.842574 avg=0.836432 VRAM=39.4GiB | 99.1% done | ETA(epoch): 636s + [Epoch 297][20/33] loss=0.861655 avg=0.838895 VRAM=39.2GiB | 99.2% done | ETA(epoch): 359s + [Epoch 297][30/33] loss=0.846086 avg=0.848952 VRAM=39.4GiB | 99.3% done | ETA(epoch): 83s + Train loss: 0.850806 (912.4s) ETA: 34min + Val loss: 0.859624 [t_0.0-0.2=1.0735 t_0.2-0.4=0.9875 t_0.4-0.6=0.8452 t_0.6-0.8=0.7091 t_0.8-1.0=0.6275] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0297 + Deleted old checkpoint: checkpoint_epoch_0294 +[MEM @ epoch 297 end] RAM: 21.5/188.4 GiB (11.4%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 298/299 (99% done) --- + [Epoch 298][10/33] loss=0.855653 avg=0.853917 VRAM=39.4GiB | 99.4% done | ETA(epoch): 636s + [Epoch 298][20/33] loss=0.851845 avg=0.857459 VRAM=39.2GiB | 99.5% done | ETA(epoch): 360s + [Epoch 298][30/33] loss=0.924752 avg=0.857974 VRAM=39.4GiB | 99.6% done | ETA(epoch): 83s + Train loss: 0.858493 (912.8s) ETA: 17min + Val loss: 0.833480 [t_0.0-0.2=1.0888 t_0.2-0.4=0.9713 t_0.4-0.6=0.8374 t_0.6-0.8=0.7410 t_0.8-1.0=0.6390] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0298 + Deleted old checkpoint: checkpoint_epoch_0295 +[MEM @ epoch 298 end] RAM: 22.8/188.4 GiB (12.1%) | VRAM: 38.5/139.8 GiB (27.5%) + +--- Epoch 299/299 (100% done) --- + [Epoch 299][10/33] loss=0.844534 avg=0.862946 VRAM=39.4GiB | 99.8% done | ETA(epoch): 636s + [Epoch 299][20/33] loss=0.757971 avg=0.850648 VRAM=39.2GiB | 99.9% done | ETA(epoch): 359s + [Epoch 299][30/33] loss=0.886613 avg=0.850008 VRAM=39.4GiB | 100.0% done | ETA(epoch): 83s + [MilestoneVis] train_0 step 9899 ✓ + [MilestoneGrid] train_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 9899 ✓ + [MilestoneGrid] train_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 9899 ✓ + [MilestoneGrid] val_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 9899 ✓ + [MilestoneGrid] val_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 9899 ✓ + [MilestoneGrid] test_0 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 9899 ✓ + [MilestoneGrid] test_1 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 9899 ✓ + [MilestoneGrid] test_2 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 9899 ✓ + [MilestoneGrid] test_3 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 9899 ✓ + [MilestoneGrid] test_4 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 9899 ✓ + [MilestoneGrid] test_5 → output/lora_256_dim_768_H200/milestone_vis/milestone_grid_test_5.png + Train loss: 0.843702 (1675.1s) ETA: 0min + Val loss: 0.840935 [t_0.0-0.2=1.0680 t_0.2-0.4=1.0441 t_0.4-0.6=0.8428 t_0.6-0.8=0.7005 t_0.8-1.0=0.6436] + Checkpoint saved: output/lora_256_dim_768_H200/checkpoint_epoch_0299 + Deleted old checkpoint: checkpoint_epoch_0296 +[MEM @ epoch 299 end] RAM: 23.2/188.4 GiB (12.3%) | VRAM: 38.5/139.8 GiB (27.5%) + [LossCurve] saved → output/lora_256_dim_768_H200/loss_curve.png + +====================================================================== +Training complete. +====================================================================== +wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: updating run metadata (0.0s) + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: Finishing up... + m wandb: uploading media/images/loss_curve_1611_74b49d145b6654d8c460.png 82.5KB/82.5KB (0.3s) +wandb: uploading wandb-summary.json 3.5KB/3.5KB (0.3s) + m m wandb: uploading media/images/loss_curve_1611_74b49d145b6654d8c460.png 82.5KB/82.5KB (0.3s) +wandb: uploading wandb-summary.json 3.5KB/3.5KB (0.3s) + m m wandb: uploading media/images/loss_curve_1611_74b49d145b6654d8c460.png 82.5KB/82.5KB (0.3s) +wandb: uploading wandb-summary.json 3.5KB/3.5KB (0.3s) + m m wandb: uploading media/images/loss_curve_1611_74b49d145b6654d8c460.png 82.5KB/82.5KB (0.3s) +wandb: uploading wandb-summary.json 3.5KB/3.5KB (0.3s) + m m wandb: uploading media/images/loss_curve_1611_74b49d145b6654d8c460.png 82.5KB/82.5KB (0.3s) +wandb: uploading wandb-summary.json 3.5KB/3.5KB (0.3s) + m m wandb: uploading history steps 1610-1611, summary (0.2s) + m wandb: uploading history steps 1610-1611, summary (0.2s) + m wandb: uploading history steps 1610-1611, summary (0.2s) + m wandb: +wandb: Run summary: +wandb: adapter_lr 5e-05 +wandb: best_val_loss 0.79418 +wandb: cpu_pct 0.6 +wandb: epoch 299 +wandb: final_train_loss 0.8437 +wandb: flops/hdc2a_per_sample_gflops 839.6179 +wandb: flops/total_per_sample_gflops 274501.79753 +wandb: flops/transformer_per_sample_gflops 273662.17962 +wandb: global_step 9896 +wandb: gpu_mem_gib 38.45154 +wandb: +10 ... +wandb: +wandb: View run lora_256_dim_768_H200-train-20260428-104100 at: https://wandb.ai/hku-xg-boost/lora_256_dim_768_H200/runs/1el5497f +wandb: View project at: https://wandb.ai/hku-xg-boost/lora_256_dim_768_H200 +wandb: Synced 4 W&B file(s), 111 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260428_104100-1el5497f/logs diff --git a/training log/trainlog(lora512dim512) b/training log/trainlog(lora512dim512) new file mode 100644 index 0000000000000000000000000000000000000000..51cf407fe8f7c96e620612f4ced5a157c8ca8b4d --- /dev/null +++ b/training log/trainlog(lora512dim512) @@ -0,0 +1,2651 @@ + +====================================================================== +Command: train_script.py --name lora_512_dim_512_H200 --batch-size 12 --adapter-lr 1e-3 --lora-lr 1e-4 --adapter-mlp --lora-rank 512 --fusion-dim 512 --seed 42 +====================================================================== +Run dir : output/lora_512_dim_512_H200 +Log file: output/lora_512_dim_512_H200/train.log +GPU: NVIDIA H200 NVL | VRAM: 139.8 GiB | PyTorch: 2.11.0+cu130 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: hkujasonjiang (hku-xg-boost) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Waiting for wandb.init()... + m wandb: Waiting for wandb.init()... + m wandb: setting up run g2ldc3sd (0.3s) + m wandb: setting up run g2ldc3sd (0.3s) + m wandb: setting up run g2ldc3sd (0.3s) + m wandb: Tracking run with wandb version 0.26.0 +wandb: Run data is saved locally in /home/xg_wang_group/SynthUrbanSAT/wandb/run-20260501_225738-g2ldc3sd +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run lora_512_dim_512_H200-train-20260501-225737 +wandb: View project at https://wandb.ai/hku-xg-boost/lora_512_dim_512_H200 +wandb: View run at https://wandb.ai/hku-xg-boost/lora_512_dim_512_H200/runs/g2ldc3sd + +Run Summary: + name lora_512_dim_512_H200 + fusion MLP (lightweight) + LoRA ON (rank=512) + depth input enabled + min-SNR weighting ON + data augmentation OFF + adapter_lr 0.001 + lora_lr 0.0001 + overfit mode False + +Final Configuration: + Paths: + transformer_path weights/flux2_dev_fp8mixed.safetensors + vae_path weights/flux2-vae.safetensors + controlnet_path weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors + dataset_dir dataset + color_map_path configs/color_map.json + output_dir output/lora_512_dim_512_H200 + text_encoder_path weights/mistral_3_small_flux2_fp8.safetensors + precomputed_embeddings output/text_embeddings_global.pt + Model: + image_size 1024 + num_classes 6 + control_in_dim 3072 + fusion_dim 512 + num_fusion_blocks 3 + num_heads 12 + num_fourier_bands 32 + boundary_threshold 0.1 + Training: + num_epochs 500 + batch_size 12 + adapter_lr 0.001 + lora_lr 0.0001 + weight_decay 0.01 + max_grad_norm 1.0 + grad_accum_steps 4 + guidance_scale 3.5 + num_workers 0 + Text Encoder: + text_seq_len 512 + text_dim 15360 + Logging: + log_interval 10 + save_every_n_epochs 5 + val_every_n_epochs 1 + WandB: + wandb_entity hku-xg-boost + wandb_project lora_512_dim_512_H200 + Resume: + resume_from (not set) +[MEM @ pre-flight] RAM: 11.5/188.4 GiB (6.1%) | VRAM: 0.0/139.8 GiB (0.0%) + +Pre-flight checks... + ✓ torch + ✓ diffusers + ✓ safetensors + ✓ Pillow + ✓ tifffile + ✓ wandb + ✓ transformers + ✓ psutil + ✓ GPU: NVIDIA H200 NVL (139.8 GiB VRAM) + ✓ transformer_path: weights/flux2_dev_fp8mixed.safetensors (35.5 GB) + ✓ vae_path: weights/flux2-vae.safetensors (0.3 GB) + ✓ controlnet_path: weights/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors (8.2 GB) + ✓ text_encoder_path: weights/mistral_3_small_flux2_fp8.safetensors (18.0 GB) + ✓ train/rgb: 400 files + ✓ train/seg: 400 files + ✓ train/depth: 400 files + ✓ val/rgb: 80 files + ✓ val/seg: 80 files + ✓ val/depth: 80 files + ✓ test/rgb: 30 files + ✓ test/seg: 30 files + ✓ test/depth: 30 files + ✓ prompt.json found + +All pre-flight checks passed. + +============================================================ + [1/8] Text Embeddings +============================================================ + Loading cached embedding from output/text_embeddings_global.pt + Loaded global text embedding from output/text_embeddings_global.pt (shape: torch.Size([512, 15360])) + +============================================================ + [2/8] Loading VAE +============================================================ + Done (1.1s), VRAM: 0.16 GiB +[MEM @ after VAE] RAM: 11.8/188.4 GiB (6.3%) | VRAM: 0.2/139.8 GiB (0.1%) + +============================================================ + [3/8] Loading Transformer +============================================================ + Dequantizing FP8 transformer weights... + Dequantized 128 FP8 tensors + Converting ComfyUI → diffusers keys... + Converted: 331 diffusers keys + Loading ControlNet weights... + ControlNet: 76 keys + Creating Flux2ControlTransformer2DModel (control_in_dim=3072)... + Skipped 2 control_img_in keys (dim mismatch): + control_img_in.bias [6144] + control_img_in.weight [6144, 260] + Missing: 2, Unexpected: 0 + Initialized control_img_in.weight [6144, 3072] on cuda + Initialized control_img_in.bias [6144] on cuda + FP8 compression: 203 frozen Linears, 67.9 → 37.9 GiB (saved 30.0 GiB) + Done (34.2s), VRAM: 37.87 GiB + Gradient checkpointing: enabled + Backbone FROZEN: all transformer params set requires_grad=False + Gradients will still propagate to HDC²A via control_context autograd +[MEM @ after Transformer] RAM: 13.6/188.4 GiB (7.2%) | VRAM: 37.9/139.8 GiB (27.1%) + +============================================================ + [4/8] Creating HDC²A Adapter +============================================================ + Fusion mode: MLP (lightweight) +HDC²A: 9.4M params +Control: 0.0M params +Total trainable: 9.4M params + +============================================================ + [4.5/8] Applying LoRA to ControlNet Control Blocks +============================================================ + LoRA rank=512, alpha=512.0, dropout=0 + LoRA control_transformer_blocks.0.attn.to_q [6144→6144] + LoRA control_transformer_blocks.0.attn.to_k [6144→6144] + LoRA control_transformer_blocks.0.attn.to_v [6144→6144] + LoRA control_transformer_blocks.0.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.0.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.1.attn.to_q [6144→6144] + LoRA control_transformer_blocks.1.attn.to_k [6144→6144] + LoRA control_transformer_blocks.1.attn.to_v [6144→6144] + LoRA control_transformer_blocks.1.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.1.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.2.attn.to_q [6144→6144] + LoRA control_transformer_blocks.2.attn.to_k [6144→6144] + LoRA control_transformer_blocks.2.attn.to_v [6144→6144] + LoRA control_transformer_blocks.2.attn.add_q_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_k_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.add_v_proj [6144→6144] + LoRA control_transformer_blocks.2.attn.to_out.0 [6144→6144] + LoRA control_transformer_blocks.3.attn.to_q [6144→6144] + LoRA control_transformer_blocks.3.attn.to_k [6144→6144] + LoRA control_transformer_blocks.3.attn.to_v [6144→6144] + LoRA control_transformer_blocks.3.attn.to_out.0 [6144→6144] + + LoRA modules injected: 25 + LoRA trainable params: 157.29M + +Parameter Statistics: + HDC²A Adapter: total=9.4M trainable=9.4M + ControlNet (frozen): total=4290.8M LoRA trainable=157.29M + Flux2 backbone: total=0.0M trainable=0.0M ✓ + ────────────────────────────────────────────────── + Total trainable: HDC²A 9.4M + LoRA 157.29M = 166.73M + +============================================================ + [5/8] Building Optimizer +============================================================ + AdamW: adapter_lr=1.00e-03, lora_lr=1.00e-04 + param_group 'adapter': 46 tensors, lr=1.00e-03 + param_group 'lora': 50 tensors, lr=1.00e-04 + Scheduler: 400 warmup steps → cosine over ~4000 steps + [6/8] Resume: skipped (no checkpoint specified) + +============================================================ + [7/8] Forward Sanity Check +============================================================ + [test 1/4] Forward pass (eval mode)... + Output shape: torch.Size([1, 4096, 128]) + Output stats: mean=0.0364, std=0.5352 + VRAM peak (forward): 68.44 GiB + [test 2/4] Loss computation (train mode)... + Loss value: 1.367426 + [test 3/4] Backward pass... + Backward completed. VRAM peak (backward): 46.93 GiB + [test 4/4] Gradient flow check... + HDC²A: 46/46 params have non-zero grad + Control: 25/50 params have non-zero grad + Top grad norms (HDC²A): + semantic_encoder.conv_stem.6.weight: 0.011536 + depth_encoder.conv_stem.6.weight: 0.010864 + W_s.weight: 0.006989 + mlp_fusion.3.weight: 0.006714 + W_d.weight: 0.006622 + Test result: PASSED +[MEM @ after test] RAM: 14.2/188.4 GiB (7.5%) | VRAM: 38.2/139.8 GiB (27.4%) +/home/xg_wang_group/miniconda/envs/flux_train/lib/python3.12/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True. + _warn_once( + +[FLOPs] Forward-only estimate (torch.profiler, approximate) + batch_size: 1 + hdc2a: 672.097 GFLOPs transformer: 274103.488 GFLOPs total: 274775.585 GFLOPs + per-sample forward FLOPs: 274775.585 GFLOPs + +============================================================ + [8/8] Loading Data +============================================================ +[Data] Data augmentation: disabled +[Data] Train: 400 samples, batch_size=12 +[Data] Train: using global text embeddings +[Data] Val: 80 samples, batch_size=12 +[Data] Test: 30 samples, batch_size=12 + +====================================================================== +Starting training: 500 epochs × 33 steps = 16500 total steps + batch_size=12, grad_accum=4, world_size=1, effective_bs=48 + adapter_lr=1.00e-03, lora_lr=1.00e-04, weight_decay=0.01 +====================================================================== + [Milestone Vis] steps: [0, 1650, 3300, 4950, 6600, 8250, 9899, 11549, 13199, 14849, 16499] + [Milestone Vis] 10 grids: train_0(5), train_1(5), val_0(5), val_1(5), test_0(5), test_1(5), test_2(5), test_3(5), test_4(5), test_5(5) + +--- Epoch 0/499 (0% done) --- + [MilestoneVis] train_0 step 0 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 0 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 0 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 0 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 0 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 0 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 0 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 0 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 0 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 0 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 0][10/33] loss=0.891481 avg=0.909754 VRAM=39.9GiB | 0.1% done | ETA(epoch): 2340s + [Epoch 0][20/33] loss=0.945387 avg=0.918693 VRAM=39.6GiB | 0.1% done | ETA(epoch): 841s + [Epoch 0][30/33] loss=0.954774 avg=0.920571 VRAM=39.9GiB | 0.2% done | ETA(epoch): 157s + Train loss: 0.918611 (1653.6s) ETA: 13775min + Val loss: 0.968127 [t_0.0-0.2=1.0920 t_0.2-0.4=1.0374 t_0.4-0.6=0.8973 t_0.6-0.8=0.7607 t_0.8-1.0=0.9604] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 1/499 (0% done) --- + [Epoch 1][10/33] loss=1.001600 avg=0.940898 VRAM=39.9GiB | 0.3% done | ETA(epoch): 636s + [Epoch 1][20/33] loss=0.983270 avg=0.924320 VRAM=39.6GiB | 0.3% done | ETA(epoch): 360s + [Epoch 1][30/33] loss=0.952024 avg=0.923574 VRAM=39.9GiB | 0.4% done | ETA(epoch): 83s + Train loss: 0.923006 (912.8s) ETA: 10935min + Val loss: 0.949847 [t_0.0-0.2=1.1041 t_0.2-0.4=1.0241 t_0.4-0.6=0.8724 t_0.6-0.8=0.7749 t_0.8-1.0=1.0480] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 16.4/188.4 GiB (8.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 2/499 (0% done) --- + [Epoch 2][10/33] loss=0.948343 avg=0.917484 VRAM=39.9GiB | 0.5% done | ETA(epoch): 636s + [Epoch 2][20/33] loss=0.946050 avg=0.919652 VRAM=39.6GiB | 0.5% done | ETA(epoch): 360s + [Epoch 2][30/33] loss=0.918973 avg=0.922699 VRAM=39.9GiB | 0.6% done | ETA(epoch): 83s + Train loss: 0.920064 (912.9s) ETA: 9978min + Val loss: 0.931116 [t_0.0-0.2=1.0831 t_0.2-0.4=1.0477 t_0.4-0.6=0.8751 t_0.6-0.8=0.7494 t_0.8-1.0=0.8988] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0002 (BEST) +[MEM @ epoch 2 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 3/499 (1% done) --- + [Epoch 3][10/33] loss=0.903626 avg=0.891454 VRAM=39.9GiB | 0.7% done | ETA(epoch): 637s + [Epoch 3][20/33] loss=0.941083 avg=0.907497 VRAM=39.6GiB | 0.7% done | ETA(epoch): 360s + [Epoch 3][30/33] loss=0.965737 avg=0.920709 VRAM=39.9GiB | 0.8% done | ETA(epoch): 83s + Train loss: 0.919916 (913.0s) ETA: 9492min + Val loss: 0.957163 [t_0.0-0.2=1.0748 t_0.2-0.4=1.0513 t_0.4-0.6=0.8678 t_0.6-0.8=0.7535 t_0.8-1.0=1.0649] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0003 + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 16.3/188.4 GiB (8.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 4/499 (1% done) --- + [Epoch 4][10/33] loss=0.938640 avg=0.897356 VRAM=39.9GiB | 0.9% done | ETA(epoch): 636s + [Epoch 4][20/33] loss=0.834284 avg=0.890444 VRAM=39.6GiB | 0.9% done | ETA(epoch): 360s + [Epoch 4][30/33] loss=0.887342 avg=0.888343 VRAM=39.9GiB | 1.0% done | ETA(epoch): 83s + Train loss: 0.889983 (912.9s) ETA: 9194min + Val loss: 0.975620 [t_0.0-0.2=1.0727 t_0.2-0.4=1.0177 t_0.4-0.6=0.8953 t_0.6-0.8=0.7482 t_0.8-1.0=1.1616] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 5/499 (1% done) --- + [Epoch 5][10/33] loss=0.857273 avg=0.892080 VRAM=39.9GiB | 1.1% done | ETA(epoch): 636s + [Epoch 5][20/33] loss=0.944316 avg=0.891152 VRAM=39.6GiB | 1.1% done | ETA(epoch): 360s + [Epoch 5][30/33] loss=0.881689 avg=0.899278 VRAM=39.9GiB | 1.2% done | ETA(epoch): 83s + Train loss: 0.898545 (912.9s) ETA: 8989min + Val loss: 0.952262 [t_0.0-0.2=1.0955 t_0.2-0.4=1.0395 t_0.4-0.6=0.8706 t_0.6-0.8=0.7350 t_0.8-1.0=1.0261] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0005 +[MEM @ epoch 5 end] RAM: 16.9/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 6/499 (1% done) --- + [Epoch 6][10/33] loss=0.958193 avg=0.893394 VRAM=39.9GiB | 1.3% done | ETA(epoch): 637s + [Epoch 6][20/33] loss=0.855579 avg=0.896556 VRAM=39.6GiB | 1.3% done | ETA(epoch): 360s + [Epoch 6][30/33] loss=0.877923 avg=0.892228 VRAM=39.9GiB | 1.4% done | ETA(epoch): 83s + Train loss: 0.892060 (912.9s) ETA: 8838min + Val loss: 0.932265 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9944 t_0.4-0.6=0.9143 t_0.6-0.8=0.7634 t_0.8-1.0=0.9489] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0006 + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 16.3/188.4 GiB (8.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 7/499 (1% done) --- + [Epoch 7][10/33] loss=0.987751 avg=0.896107 VRAM=39.9GiB | 1.5% done | ETA(epoch): 636s + [Epoch 7][20/33] loss=0.888150 avg=0.899243 VRAM=39.6GiB | 1.5% done | ETA(epoch): 360s + [Epoch 7][30/33] loss=0.903711 avg=0.902012 VRAM=39.9GiB | 1.6% done | ETA(epoch): 83s + Train loss: 0.898217 (913.1s) ETA: 8721min + Val loss: 0.960744 [t_0.0-0.2=1.0787 t_0.2-0.4=0.9978 t_0.4-0.6=0.8429 t_0.6-0.8=0.7355 t_0.8-1.0=0.9241] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 8/499 (2% done) --- + [Epoch 8][10/33] loss=0.883895 avg=0.890435 VRAM=39.9GiB | 1.7% done | ETA(epoch): 636s + [Epoch 8][20/33] loss=0.903729 avg=0.899441 VRAM=39.6GiB | 1.7% done | ETA(epoch): 360s + [Epoch 8][30/33] loss=0.940184 avg=0.892995 VRAM=39.9GiB | 1.8% done | ETA(epoch): 83s + Train loss: 0.891539 (913.3s) ETA: 8627min + Val loss: 0.907683 [t_0.0-0.2=1.0922 t_0.2-0.4=0.9710 t_0.4-0.6=0.8481 t_0.6-0.8=0.7485 t_0.8-1.0=0.9085] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0008 (BEST) + Deleted old checkpoint: checkpoint_epoch_0002 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 9/499 (2% done) --- + [Epoch 9][10/33] loss=0.843942 avg=0.880777 VRAM=39.9GiB | 1.9% done | ETA(epoch): 637s + [Epoch 9][20/33] loss=0.919608 avg=0.895242 VRAM=39.6GiB | 1.9% done | ETA(epoch): 360s + [Epoch 9][30/33] loss=0.852294 avg=0.894961 VRAM=39.9GiB | 2.0% done | ETA(epoch): 83s + Train loss: 0.889906 (913.5s) ETA: 8549min + Val loss: 0.884450 [t_0.0-0.2=1.0796 t_0.2-0.4=0.9957 t_0.4-0.6=0.8798 t_0.6-0.8=0.7569 t_0.8-1.0=0.7976] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0009 (BEST) + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 10/499 (2% done) --- + [Epoch 10][10/33] loss=0.899678 avg=0.888796 VRAM=39.9GiB | 2.1% done | ETA(epoch): 636s + [Epoch 10][20/33] loss=0.907937 avg=0.886996 VRAM=39.6GiB | 2.1% done | ETA(epoch): 359s + [Epoch 10][30/33] loss=0.847857 avg=0.891973 VRAM=39.9GiB | 2.2% done | ETA(epoch): 83s + Train loss: 0.892928 (912.7s) ETA: 8481min + Val loss: 0.929801 [t_0.0-0.2=1.0746 t_0.2-0.4=1.0277 t_0.4-0.6=0.8858 t_0.6-0.8=0.7460 t_0.8-1.0=0.8283] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0010 + Deleted old checkpoint: checkpoint_epoch_0007 +[MEM @ epoch 10 end] RAM: 16.7/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 11/499 (2% done) --- + [Epoch 11][10/33] loss=0.927559 avg=0.880655 VRAM=39.9GiB | 2.3% done | ETA(epoch): 637s + [Epoch 11][20/33] loss=0.828380 avg=0.880257 VRAM=39.6GiB | 2.3% done | ETA(epoch): 360s + [Epoch 11][30/33] loss=0.779024 avg=0.881886 VRAM=39.9GiB | 2.4% done | ETA(epoch): 83s + Train loss: 0.884254 (913.3s) ETA: 8422min + Val loss: 0.901265 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9975 t_0.4-0.6=0.8525 t_0.6-0.8=0.7674 t_0.8-1.0=0.7768] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 12/499 (2% done) --- + [Epoch 12][10/33] loss=0.927146 avg=0.906629 VRAM=39.9GiB | 2.5% done | ETA(epoch): 636s + [Epoch 12][20/33] loss=0.839455 avg=0.892148 VRAM=39.6GiB | 2.5% done | ETA(epoch): 360s + [Epoch 12][30/33] loss=0.823719 avg=0.887672 VRAM=39.9GiB | 2.6% done | ETA(epoch): 83s + Train loss: 0.888696 (913.1s) ETA: 8370min + Val loss: 0.917111 [t_0.0-0.2=1.0734 t_0.2-0.4=1.0374 t_0.4-0.6=0.8507 t_0.6-0.8=0.7307 t_0.8-1.0=0.8062] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0012 +[MEM @ epoch 12 end] RAM: 16.8/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 13/499 (3% done) --- + [Epoch 13][10/33] loss=0.916510 avg=0.882295 VRAM=39.9GiB | 2.7% done | ETA(epoch): 636s + [Epoch 13][20/33] loss=0.921553 avg=0.880447 VRAM=39.6GiB | 2.7% done | ETA(epoch): 360s + [Epoch 13][30/33] loss=0.902495 avg=0.885276 VRAM=39.9GiB | 2.8% done | ETA(epoch): 83s + Train loss: 0.881667 (913.3s) ETA: 8323min + Val loss: 0.909114 [t_0.0-0.2=1.0814 t_0.2-0.4=1.0141 t_0.4-0.6=0.8269 t_0.6-0.8=0.7596 t_0.8-1.0=0.7489] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 14/499 (3% done) --- + [Epoch 14][10/33] loss=0.883063 avg=0.882344 VRAM=39.9GiB | 2.9% done | ETA(epoch): 637s + [Epoch 14][20/33] loss=0.863938 avg=0.875184 VRAM=39.6GiB | 2.9% done | ETA(epoch): 360s + [Epoch 14][30/33] loss=0.822612 avg=0.871680 VRAM=39.9GiB | 3.0% done | ETA(epoch): 83s + Train loss: 0.872031 (913.7s) ETA: 8280min + Val loss: 0.888625 [t_0.0-0.2=1.0676 t_0.2-0.4=1.0030 t_0.4-0.6=0.8330 t_0.6-0.8=0.7474 t_0.8-1.0=0.7362] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0014 + Deleted old checkpoint: checkpoint_epoch_0011 +[MEM @ epoch 14 end] RAM: 15.7/188.4 GiB (8.4%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 15/499 (3% done) --- + [Epoch 15][10/33] loss=0.941988 avg=0.894835 VRAM=39.9GiB | 3.1% done | ETA(epoch): 636s + [Epoch 15][20/33] loss=0.892625 avg=0.892205 VRAM=39.6GiB | 3.1% done | ETA(epoch): 360s + [Epoch 15][30/33] loss=0.928833 avg=0.884613 VRAM=39.9GiB | 3.2% done | ETA(epoch): 83s + Train loss: 0.884417 (913.7s) ETA: 8240min + Val loss: 0.875289 [t_0.0-0.2=1.0626 t_0.2-0.4=1.0071 t_0.4-0.6=0.8720 t_0.6-0.8=0.7195 t_0.8-1.0=0.7470] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0015 (BEST) + Deleted old checkpoint: checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 16/499 (3% done) --- + [Epoch 16][10/33] loss=0.864330 avg=0.909735 VRAM=39.9GiB | 3.3% done | ETA(epoch): 637s + [Epoch 16][20/33] loss=0.949419 avg=0.890971 VRAM=39.6GiB | 3.3% done | ETA(epoch): 360s + [Epoch 16][30/33] loss=0.852868 avg=0.882280 VRAM=39.9GiB | 3.4% done | ETA(epoch): 83s + Train loss: 0.881939 (913.9s) ETA: 8204min + Val loss: 0.869829 [t_0.0-0.2=1.0839 t_0.2-0.4=0.9783 t_0.4-0.6=0.8550 t_0.6-0.8=0.7322 t_0.8-1.0=0.7250] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0016 (BEST) + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 17/499 (3% done) --- + [Epoch 17][10/33] loss=0.827899 avg=0.899998 VRAM=39.9GiB | 3.5% done | ETA(epoch): 637s + [Epoch 17][20/33] loss=0.844503 avg=0.878057 VRAM=39.6GiB | 3.5% done | ETA(epoch): 360s + [Epoch 17][30/33] loss=0.864697 avg=0.874469 VRAM=39.9GiB | 3.6% done | ETA(epoch): 83s + Train loss: 0.876995 (914.4s) ETA: 8170min + Val loss: 0.870084 [t_0.0-0.2=1.0716 t_0.2-0.4=1.0112 t_0.4-0.6=0.8719 t_0.6-0.8=0.7283 t_0.8-1.0=0.7282] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 18/499 (4% done) --- + [Epoch 18][10/33] loss=0.868292 avg=0.887355 VRAM=39.9GiB | 3.7% done | ETA(epoch): 637s + [Epoch 18][20/33] loss=0.942042 avg=0.882123 VRAM=39.6GiB | 3.7% done | ETA(epoch): 360s + [Epoch 18][30/33] loss=0.867598 avg=0.884186 VRAM=39.9GiB | 3.8% done | ETA(epoch): 83s + Train loss: 0.881541 (914.2s) ETA: 8137min + Val loss: 0.876816 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0279 t_0.4-0.6=0.8444 t_0.6-0.8=0.7382 t_0.8-1.0=0.7446] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 19/499 (4% done) --- + [Epoch 19][10/33] loss=0.861149 avg=0.870791 VRAM=39.9GiB | 3.9% done | ETA(epoch): 637s + [Epoch 19][20/33] loss=0.793957 avg=0.871903 VRAM=39.6GiB | 3.9% done | ETA(epoch): 360s + [Epoch 19][30/33] loss=0.848821 avg=0.879712 VRAM=39.9GiB | 4.0% done | ETA(epoch): 83s + Train loss: 0.879227 (914.2s) ETA: 8107min + Val loss: 0.886180 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9768 t_0.4-0.6=0.8817 t_0.6-0.8=0.7329 t_0.8-1.0=0.7390] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0019 +[MEM @ epoch 19 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 20/499 (4% done) --- + [Epoch 20][10/33] loss=0.868818 avg=0.867955 VRAM=39.9GiB | 4.1% done | ETA(epoch): 637s + [Epoch 20][20/33] loss=0.930939 avg=0.883393 VRAM=39.6GiB | 4.1% done | ETA(epoch): 360s + [Epoch 20][30/33] loss=0.960967 avg=0.882716 VRAM=39.9GiB | 4.2% done | ETA(epoch): 83s + Train loss: 0.881687 (914.9s) ETA: 8078min + Val loss: 0.891290 [t_0.0-0.2=1.0880 t_0.2-0.4=1.0032 t_0.4-0.6=0.8489 t_0.6-0.8=0.7101 t_0.8-1.0=0.7540] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0020 + Deleted old checkpoint: checkpoint_epoch_0017 +[MEM @ epoch 20 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 21/499 (4% done) --- + [Epoch 21][10/33] loss=0.813565 avg=0.871024 VRAM=39.9GiB | 4.3% done | ETA(epoch): 637s + [Epoch 21][20/33] loss=0.829047 avg=0.870078 VRAM=39.6GiB | 4.3% done | ETA(epoch): 360s + [Epoch 21][30/33] loss=0.958727 avg=0.872502 VRAM=39.9GiB | 4.4% done | ETA(epoch): 83s + Train loss: 0.870486 (914.5s) ETA: 8049min + Val loss: 0.869683 [t_0.0-0.2=1.0758 t_0.2-0.4=1.0075 t_0.4-0.6=0.8351 t_0.6-0.8=0.7219 t_0.8-1.0=0.7259] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0021 (BEST) + Deleted old checkpoint: checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 22/499 (4% done) --- + [Epoch 22][10/33] loss=0.809126 avg=0.883518 VRAM=39.9GiB | 4.5% done | ETA(epoch): 637s + [Epoch 22][20/33] loss=0.937889 avg=0.872731 VRAM=39.6GiB | 4.5% done | ETA(epoch): 360s + [Epoch 22][30/33] loss=0.867730 avg=0.870224 VRAM=39.9GiB | 4.6% done | ETA(epoch): 83s + Train loss: 0.866296 (914.0s) ETA: 8022min + Val loss: 0.837361 [t_0.0-0.2=1.0885 t_0.2-0.4=0.9649 t_0.4-0.6=0.8343 t_0.6-0.8=0.7320 t_0.8-1.0=0.7160] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0022 (BEST) + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 23/499 (5% done) --- + [Epoch 23][10/33] loss=0.831021 avg=0.879190 VRAM=39.9GiB | 4.7% done | ETA(epoch): 637s + [Epoch 23][20/33] loss=0.874084 avg=0.880126 VRAM=39.6GiB | 4.7% done | ETA(epoch): 360s + [Epoch 23][30/33] loss=0.824500 avg=0.870824 VRAM=39.9GiB | 4.8% done | ETA(epoch): 83s + Train loss: 0.868085 (914.6s) ETA: 7996min + Val loss: 0.885587 [t_0.0-0.2=1.0743 t_0.2-0.4=1.0370 t_0.4-0.6=0.8662 t_0.6-0.8=0.7517 t_0.8-1.0=0.7220] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0023 + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 24/499 (5% done) --- + [Epoch 24][10/33] loss=0.932582 avg=0.880060 VRAM=39.9GiB | 4.9% done | ETA(epoch): 637s + [Epoch 24][20/33] loss=0.866487 avg=0.879649 VRAM=39.6GiB | 4.9% done | ETA(epoch): 360s + [Epoch 24][30/33] loss=0.908041 avg=0.882462 VRAM=39.9GiB | 5.0% done | ETA(epoch): 83s + Train loss: 0.881079 (913.9s) ETA: 7971min + Val loss: 0.871187 [t_0.0-0.2=1.0818 t_0.2-0.4=0.9960 t_0.4-0.6=0.8578 t_0.6-0.8=0.7259 t_0.8-1.0=0.7206] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 15.4/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 25/499 (5% done) --- + [Epoch 25][10/33] loss=0.864952 avg=0.882687 VRAM=39.9GiB | 5.1% done | ETA(epoch): 636s + [Epoch 25][20/33] loss=0.840555 avg=0.872183 VRAM=39.6GiB | 5.1% done | ETA(epoch): 360s + [Epoch 25][30/33] loss=0.804241 avg=0.870673 VRAM=39.9GiB | 5.2% done | ETA(epoch): 83s + Train loss: 0.872462 (913.8s) ETA: 7946min + Val loss: 0.898341 [t_0.0-0.2=1.0809 t_0.2-0.4=0.9905 t_0.4-0.6=0.8068 t_0.6-0.8=0.7348 t_0.8-1.0=0.7370] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0025 +[MEM @ epoch 25 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 26/499 (5% done) --- + [Epoch 26][10/33] loss=0.871916 avg=0.847161 VRAM=39.9GiB | 5.3% done | ETA(epoch): 637s + [Epoch 26][20/33] loss=0.849642 avg=0.855379 VRAM=39.6GiB | 5.3% done | ETA(epoch): 360s + [Epoch 26][30/33] loss=0.920842 avg=0.862634 VRAM=39.9GiB | 5.4% done | ETA(epoch): 83s + Train loss: 0.861953 (913.5s) ETA: 7921min + Val loss: 0.882696 [t_0.0-0.2=1.0866 t_0.2-0.4=0.9839 t_0.4-0.6=0.8626 t_0.6-0.8=0.7426 t_0.8-1.0=0.6954] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 27/499 (5% done) --- + [Epoch 27][10/33] loss=0.897991 avg=0.868994 VRAM=39.9GiB | 5.5% done | ETA(epoch): 637s + [Epoch 27][20/33] loss=0.772266 avg=0.873174 VRAM=39.6GiB | 5.5% done | ETA(epoch): 360s + [Epoch 27][30/33] loss=0.910098 avg=0.872285 VRAM=39.9GiB | 5.6% done | ETA(epoch): 83s + Train loss: 0.870933 (914.4s) ETA: 7898min + Val loss: 0.912268 [t_0.0-0.2=1.0714 t_0.2-0.4=1.0024 t_0.4-0.6=0.8520 t_0.6-0.8=0.7518 t_0.8-1.0=0.7162] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 28/499 (6% done) --- + [Epoch 28][10/33] loss=0.843573 avg=0.859300 VRAM=39.9GiB | 5.7% done | ETA(epoch): 637s + [Epoch 28][20/33] loss=0.858612 avg=0.863125 VRAM=39.6GiB | 5.7% done | ETA(epoch): 360s + [Epoch 28][30/33] loss=0.911408 avg=0.879266 VRAM=39.9GiB | 5.8% done | ETA(epoch): 83s + Train loss: 0.880837 (914.1s) ETA: 7875min + Val loss: 0.884992 [t_0.0-0.2=1.0873 t_0.2-0.4=1.0220 t_0.4-0.6=0.8288 t_0.6-0.8=0.7400 t_0.8-1.0=0.7134] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 29/499 (6% done) --- + [Epoch 29][10/33] loss=0.862821 avg=0.853833 VRAM=39.9GiB | 5.9% done | ETA(epoch): 636s + [Epoch 29][20/33] loss=0.804093 avg=0.847266 VRAM=39.6GiB | 5.9% done | ETA(epoch): 360s + [Epoch 29][30/33] loss=0.931834 avg=0.854734 VRAM=39.9GiB | 6.0% done | ETA(epoch): 83s + Train loss: 0.854533 (913.3s) ETA: 7852min + Val loss: 0.867937 [t_0.0-0.2=1.0854 t_0.2-0.4=0.9734 t_0.4-0.6=0.8404 t_0.6-0.8=0.7456 t_0.8-1.0=0.7109] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0029 + Deleted old checkpoint: checkpoint_epoch_0026 +[MEM @ epoch 29 end] RAM: 15.4/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 30/499 (6% done) --- + [Epoch 30][10/33] loss=0.885843 avg=0.864710 VRAM=39.9GiB | 6.1% done | ETA(epoch): 637s + [Epoch 30][20/33] loss=0.869731 avg=0.862848 VRAM=39.6GiB | 6.1% done | ETA(epoch): 360s + [Epoch 30][30/33] loss=0.840003 avg=0.865150 VRAM=39.9GiB | 6.2% done | ETA(epoch): 83s + Train loss: 0.867515 (914.4s) ETA: 7830min + Val loss: 0.876261 [t_0.0-0.2=1.0626 t_0.2-0.4=0.9746 t_0.4-0.6=0.8607 t_0.6-0.8=0.7297 t_0.8-1.0=0.7046] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 31/499 (6% done) --- + [Epoch 31][10/33] loss=0.923982 avg=0.859315 VRAM=39.9GiB | 6.3% done | ETA(epoch): 637s + [Epoch 31][20/33] loss=0.822859 avg=0.856006 VRAM=39.6GiB | 6.3% done | ETA(epoch): 360s + [Epoch 31][30/33] loss=0.938471 avg=0.857165 VRAM=39.9GiB | 6.4% done | ETA(epoch): 83s + Train loss: 0.856356 (914.7s) ETA: 7808min + Val loss: 0.898961 [t_0.0-0.2=1.0693 t_0.2-0.4=1.0029 t_0.4-0.6=0.8470 t_0.6-0.8=0.7031 t_0.8-1.0=0.6810] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 32/499 (6% done) --- + [Epoch 32][10/33] loss=0.853587 avg=0.866950 VRAM=39.9GiB | 6.5% done | ETA(epoch): 637s + [Epoch 32][20/33] loss=0.973356 avg=0.860842 VRAM=39.6GiB | 6.5% done | ETA(epoch): 360s + [Epoch 32][30/33] loss=0.905107 avg=0.865294 VRAM=39.9GiB | 6.6% done | ETA(epoch): 83s + Train loss: 0.872382 (914.0s) ETA: 7786min + Val loss: 0.839277 [t_0.0-0.2=1.0785 t_0.2-0.4=0.9978 t_0.4-0.6=0.8468 t_0.6-0.8=0.7343 t_0.8-1.0=0.7037] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 33/499 (7% done) --- + [Epoch 33][10/33] loss=0.977750 avg=0.906305 VRAM=39.9GiB | 6.7% done | ETA(epoch): 637s + [Epoch 33][20/33] loss=0.761568 avg=0.889314 VRAM=39.6GiB | 6.7% done | ETA(epoch): 360s + [Epoch 33][30/33] loss=0.773220 avg=0.875435 VRAM=39.9GiB | 6.8% done | ETA(epoch): 83s + Train loss: 0.877425 (914.7s) ETA: 7765min + Val loss: 0.891841 [t_0.0-0.2=1.0650 t_0.2-0.4=1.0052 t_0.4-0.6=0.8745 t_0.6-0.8=0.7392 t_0.8-1.0=0.6725] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 34/499 (7% done) --- + [Epoch 34][10/33] loss=0.927709 avg=0.864687 VRAM=39.9GiB | 6.9% done | ETA(epoch): 638s + [Epoch 34][20/33] loss=0.846325 avg=0.860864 VRAM=39.6GiB | 6.9% done | ETA(epoch): 360s + [Epoch 34][30/33] loss=0.899926 avg=0.864092 VRAM=39.9GiB | 7.0% done | ETA(epoch): 83s + Train loss: 0.867564 (914.8s) ETA: 7744min + Val loss: 0.866928 [t_0.0-0.2=1.0627 t_0.2-0.4=1.0041 t_0.4-0.6=0.8351 t_0.6-0.8=0.7297 t_0.8-1.0=0.6843] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 35/499 (7% done) --- + [Epoch 35][10/33] loss=0.912250 avg=0.869888 VRAM=39.9GiB | 7.1% done | ETA(epoch): 637s + [Epoch 35][20/33] loss=0.889770 avg=0.858351 VRAM=39.6GiB | 7.1% done | ETA(epoch): 360s + [Epoch 35][30/33] loss=0.867559 avg=0.858816 VRAM=39.9GiB | 7.2% done | ETA(epoch): 83s + Train loss: 0.860671 (914.5s) ETA: 7724min + Val loss: 0.885792 [t_0.0-0.2=1.0739 t_0.2-0.4=1.0109 t_0.4-0.6=0.8297 t_0.6-0.8=0.7123 t_0.8-1.0=0.6996] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 36/499 (7% done) --- + [Epoch 36][10/33] loss=0.768880 avg=0.884692 VRAM=39.9GiB | 7.3% done | ETA(epoch): 637s + [Epoch 36][20/33] loss=0.894054 avg=0.896147 VRAM=39.6GiB | 7.3% done | ETA(epoch): 360s + [Epoch 36][30/33] loss=0.811007 avg=0.879519 VRAM=39.9GiB | 7.4% done | ETA(epoch): 83s + Train loss: 0.879361 (914.6s) ETA: 7703min + Val loss: 0.863784 [t_0.0-0.2=1.0865 t_0.2-0.4=1.0263 t_0.4-0.6=0.8689 t_0.6-0.8=0.7262 t_0.8-1.0=0.6866] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 15.8/188.4 GiB (8.4%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 37/499 (7% done) --- + [Epoch 37][10/33] loss=0.789282 avg=0.871329 VRAM=39.9GiB | 7.5% done | ETA(epoch): 637s + [Epoch 37][20/33] loss=0.884771 avg=0.867492 VRAM=39.6GiB | 7.5% done | ETA(epoch): 360s + [Epoch 37][30/33] loss=0.877099 avg=0.879844 VRAM=39.9GiB | 7.6% done | ETA(epoch): 83s + Train loss: 0.881903 (914.4s) ETA: 7683min + Val loss: 0.873420 [t_0.0-0.2=1.0778 t_0.2-0.4=0.9946 t_0.4-0.6=0.8637 t_0.6-0.8=0.7237 t_0.8-1.0=0.6801] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 38/499 (8% done) --- + [Epoch 38][10/33] loss=0.822684 avg=0.873192 VRAM=39.9GiB | 7.7% done | ETA(epoch): 637s + [Epoch 38][20/33] loss=0.849552 avg=0.875009 VRAM=39.6GiB | 7.7% done | ETA(epoch): 360s + [Epoch 38][30/33] loss=0.818753 avg=0.864554 VRAM=39.9GiB | 7.8% done | ETA(epoch): 83s + Train loss: 0.863158 (914.8s) ETA: 7663min + Val loss: 0.880381 [t_0.0-0.2=1.0775 t_0.2-0.4=1.0040 t_0.4-0.6=0.8598 t_0.6-0.8=0.7509 t_0.8-1.0=0.7001] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 39/499 (8% done) --- + [Epoch 39][10/33] loss=0.791328 avg=0.847417 VRAM=39.9GiB | 7.9% done | ETA(epoch): 638s + [Epoch 39][20/33] loss=0.921214 avg=0.864530 VRAM=39.6GiB | 7.9% done | ETA(epoch): 361s + [Epoch 39][30/33] loss=0.748168 avg=0.862021 VRAM=39.9GiB | 8.0% done | ETA(epoch): 83s + Train loss: 0.857979 (915.6s) ETA: 7644min + Val loss: 0.881366 [t_0.0-0.2=1.0907 t_0.2-0.4=0.9986 t_0.4-0.6=0.8600 t_0.6-0.8=0.7335 t_0.8-1.0=0.6814] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 40/499 (8% done) --- + [Epoch 40][10/33] loss=0.948728 avg=0.884425 VRAM=39.9GiB | 8.1% done | ETA(epoch): 638s + [Epoch 40][20/33] loss=0.868039 avg=0.880776 VRAM=39.6GiB | 8.1% done | ETA(epoch): 360s + [Epoch 40][30/33] loss=0.952025 avg=0.880868 VRAM=39.9GiB | 8.2% done | ETA(epoch): 83s + Train loss: 0.879301 (914.8s) ETA: 7624min + Val loss: 0.860868 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9751 t_0.4-0.6=0.8566 t_0.6-0.8=0.7357 t_0.8-1.0=0.6875] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 41/499 (8% done) --- + [Epoch 41][10/33] loss=0.884895 avg=0.845971 VRAM=39.9GiB | 8.3% done | ETA(epoch): 637s + [Epoch 41][20/33] loss=0.845332 avg=0.866215 VRAM=39.6GiB | 8.3% done | ETA(epoch): 360s + [Epoch 41][30/33] loss=0.865778 avg=0.870054 VRAM=39.9GiB | 8.4% done | ETA(epoch): 83s + Train loss: 0.870877 (914.4s) ETA: 7605min + Val loss: 0.890586 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9670 t_0.4-0.6=0.8869 t_0.6-0.8=0.7117 t_0.8-1.0=0.7273] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0041 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 42/499 (8% done) --- + [Epoch 42][10/33] loss=0.778210 avg=0.857276 VRAM=39.9GiB | 8.5% done | ETA(epoch): 637s + [Epoch 42][20/33] loss=0.798449 avg=0.857693 VRAM=39.6GiB | 8.5% done | ETA(epoch): 360s + [Epoch 42][30/33] loss=0.860968 avg=0.857749 VRAM=39.9GiB | 8.6% done | ETA(epoch): 83s + Train loss: 0.861199 (914.1s) ETA: 7585min + Val loss: 0.874312 [t_0.0-0.2=1.0753 t_0.2-0.4=0.9719 t_0.4-0.6=0.8939 t_0.6-0.8=0.7259 t_0.8-1.0=0.6904] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 43/499 (9% done) --- + [Epoch 43][10/33] loss=0.913588 avg=0.883097 VRAM=39.9GiB | 8.7% done | ETA(epoch): 637s + [Epoch 43][20/33] loss=0.853665 avg=0.877445 VRAM=39.6GiB | 8.7% done | ETA(epoch): 360s + [Epoch 43][30/33] loss=0.879320 avg=0.878076 VRAM=39.9GiB | 8.8% done | ETA(epoch): 83s + Train loss: 0.879401 (914.5s) ETA: 7566min + Val loss: 0.871021 [t_0.0-0.2=1.0636 t_0.2-0.4=1.0409 t_0.4-0.6=0.8502 t_0.6-0.8=0.7248 t_0.8-1.0=0.7042] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0043 + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 44/499 (9% done) --- + [Epoch 44][10/33] loss=0.885388 avg=0.882034 VRAM=39.9GiB | 8.9% done | ETA(epoch): 637s + [Epoch 44][20/33] loss=0.836654 avg=0.875778 VRAM=39.6GiB | 8.9% done | ETA(epoch): 360s + [Epoch 44][30/33] loss=0.894169 avg=0.874844 VRAM=39.9GiB | 9.0% done | ETA(epoch): 83s + Train loss: 0.873505 (915.0s) ETA: 7547min + Val loss: 0.872656 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9929 t_0.4-0.6=0.8310 t_0.6-0.8=0.7555 t_0.8-1.0=0.6986] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 45/499 (9% done) --- + [Epoch 45][10/33] loss=0.891702 avg=0.889709 VRAM=39.9GiB | 9.1% done | ETA(epoch): 637s + [Epoch 45][20/33] loss=0.818120 avg=0.875454 VRAM=39.6GiB | 9.1% done | ETA(epoch): 360s + [Epoch 45][30/33] loss=0.821730 avg=0.871409 VRAM=39.9GiB | 9.2% done | ETA(epoch): 83s + Train loss: 0.874424 (915.3s) ETA: 7528min + Val loss: 0.908788 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9791 t_0.4-0.6=0.8964 t_0.6-0.8=0.7568 t_0.8-1.0=0.6652] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 46/499 (9% done) --- + [Epoch 46][10/33] loss=0.871128 avg=0.866973 VRAM=39.9GiB | 9.3% done | ETA(epoch): 637s + [Epoch 46][20/33] loss=0.805865 avg=0.861741 VRAM=39.6GiB | 9.3% done | ETA(epoch): 360s + [Epoch 46][30/33] loss=0.959012 avg=0.874528 VRAM=39.9GiB | 9.4% done | ETA(epoch): 83s + Train loss: 0.872890 (914.4s) ETA: 7509min + Val loss: 0.886388 [t_0.0-0.2=1.0679 t_0.2-0.4=1.0074 t_0.4-0.6=0.8579 t_0.6-0.8=0.7340 t_0.8-1.0=0.6829] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0046 + Deleted old checkpoint: checkpoint_epoch_0043 +[MEM @ epoch 46 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 47/499 (9% done) --- + [Epoch 47][10/33] loss=0.824704 avg=0.855570 VRAM=39.9GiB | 9.5% done | ETA(epoch): 638s + [Epoch 47][20/33] loss=0.848106 avg=0.864077 VRAM=39.6GiB | 9.5% done | ETA(epoch): 360s + [Epoch 47][30/33] loss=0.889072 avg=0.871601 VRAM=39.9GiB | 9.6% done | ETA(epoch): 83s + Train loss: 0.870884 (914.6s) ETA: 7491min + Val loss: 0.853438 [t_0.0-0.2=1.0666 t_0.2-0.4=0.9811 t_0.4-0.6=0.8419 t_0.6-0.8=0.7289 t_0.8-1.0=0.6789] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 48/499 (10% done) --- + [Epoch 48][10/33] loss=0.945105 avg=0.868589 VRAM=39.9GiB | 9.7% done | ETA(epoch): 637s + [Epoch 48][20/33] loss=0.875008 avg=0.866833 VRAM=39.6GiB | 9.7% done | ETA(epoch): 360s + [Epoch 48][30/33] loss=0.959488 avg=0.873464 VRAM=39.9GiB | 9.8% done | ETA(epoch): 83s + Train loss: 0.874936 (914.7s) ETA: 7472min + Val loss: 0.851537 [t_0.0-0.2=1.0838 t_0.2-0.4=0.9953 t_0.4-0.6=0.8416 t_0.6-0.8=0.7547 t_0.8-1.0=0.6870] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 49/499 (10% done) --- + [Epoch 49][10/33] loss=0.860476 avg=0.869808 VRAM=39.9GiB | 9.9% done | ETA(epoch): 637s + [Epoch 49][20/33] loss=0.894785 avg=0.865400 VRAM=39.6GiB | 9.9% done | ETA(epoch): 360s + [Epoch 49][30/33] loss=0.933562 avg=0.867048 VRAM=39.9GiB | 10.0% done | ETA(epoch): 83s + Train loss: 0.866863 (914.0s) ETA: 7453min + Val loss: 0.862943 [t_0.0-0.2=1.0649 t_0.2-0.4=1.0203 t_0.4-0.6=0.8646 t_0.6-0.8=0.7299 t_0.8-1.0=0.6856] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 50/499 (10% done) --- + [MilestoneVis] train_0 step 1650 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 1650 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 1650 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 1650 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 1650 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 1650 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 1650 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 1650 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 1650 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 1650 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 50][10/33] loss=0.832496 avg=0.858423 VRAM=39.9GiB | 10.1% done | ETA(epoch): 2366s + [Epoch 50][20/33] loss=0.846092 avg=0.867679 VRAM=39.6GiB | 10.1% done | ETA(epoch): 849s + [Epoch 50][30/33] loss=0.870001 avg=0.863866 VRAM=39.9GiB | 10.2% done | ETA(epoch): 158s + Train loss: 0.860122 (1667.9s) ETA: 7545min + Val loss: 0.865683 [t_0.0-0.2=1.0531 t_0.2-0.4=0.9994 t_0.4-0.6=0.8726 t_0.6-0.8=0.7254 t_0.8-1.0=0.6748] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0050 + Deleted old checkpoint: checkpoint_epoch_0047 +[MEM @ epoch 50 end] RAM: 16.3/188.4 GiB (8.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 51/499 (10% done) --- + [Epoch 51][10/33] loss=0.936222 avg=0.873568 VRAM=39.9GiB | 10.3% done | ETA(epoch): 639s + [Epoch 51][20/33] loss=0.907504 avg=0.867141 VRAM=39.6GiB | 10.3% done | ETA(epoch): 361s + [Epoch 51][30/33] loss=0.860532 avg=0.860485 VRAM=39.9GiB | 10.4% done | ETA(epoch): 83s + Train loss: 0.869713 (917.1s) ETA: 7525min + Val loss: 0.922365 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9956 t_0.4-0.6=0.8398 t_0.6-0.8=0.7304 t_0.8-1.0=0.6873] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0051 + Deleted old checkpoint: checkpoint_epoch_0048 +[MEM @ epoch 51 end] RAM: 16.4/188.4 GiB (8.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 52/499 (10% done) --- + [Epoch 52][10/33] loss=0.904527 avg=0.860485 VRAM=39.9GiB | 10.5% done | ETA(epoch): 638s + [Epoch 52][20/33] loss=0.914864 avg=0.864205 VRAM=39.6GiB | 10.5% done | ETA(epoch): 361s + [Epoch 52][30/33] loss=0.861882 avg=0.864104 VRAM=39.9GiB | 10.6% done | ETA(epoch): 83s + Train loss: 0.863446 (915.3s) ETA: 7505min + Val loss: 0.848961 [t_0.0-0.2=1.0758 t_0.2-0.4=1.0017 t_0.4-0.6=0.8261 t_0.6-0.8=0.7353 t_0.8-1.0=0.6778] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0052 + Deleted old checkpoint: checkpoint_epoch_0049 +[MEM @ epoch 52 end] RAM: 16.5/188.4 GiB (8.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 53/499 (11% done) --- + [Epoch 53][10/33] loss=0.821300 avg=0.876710 VRAM=39.9GiB | 10.7% done | ETA(epoch): 638s + [Epoch 53][20/33] loss=0.910227 avg=0.887624 VRAM=39.6GiB | 10.7% done | ETA(epoch): 360s + [Epoch 53][30/33] loss=0.789311 avg=0.874433 VRAM=39.9GiB | 10.8% done | ETA(epoch): 83s + Train loss: 0.876527 (914.7s) ETA: 7484min + Val loss: 0.848221 [t_0.0-0.2=1.0602 t_0.2-0.4=0.9991 t_0.4-0.6=0.8675 t_0.6-0.8=0.7344 t_0.8-1.0=0.6590] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0053 + Deleted old checkpoint: checkpoint_epoch_0050 +[MEM @ epoch 53 end] RAM: 15.5/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 54/499 (11% done) --- + [Epoch 54][10/33] loss=0.928534 avg=0.864185 VRAM=39.9GiB | 10.9% done | ETA(epoch): 637s + [Epoch 54][20/33] loss=0.838778 avg=0.863481 VRAM=39.6GiB | 10.9% done | ETA(epoch): 360s + [Epoch 54][30/33] loss=0.898319 avg=0.867089 VRAM=39.9GiB | 11.0% done | ETA(epoch): 83s + Train loss: 0.866073 (912.9s) ETA: 7464min + Val loss: 0.879219 [t_0.0-0.2=1.0661 t_0.2-0.4=0.9987 t_0.4-0.6=0.8527 t_0.6-0.8=0.7548 t_0.8-1.0=0.6687] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0054 + Deleted old checkpoint: checkpoint_epoch_0051 +[MEM @ epoch 54 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 55/499 (11% done) --- + [Epoch 55][10/33] loss=0.899585 avg=0.867254 VRAM=39.9GiB | 11.1% done | ETA(epoch): 636s + [Epoch 55][20/33] loss=0.886661 avg=0.861744 VRAM=39.6GiB | 11.1% done | ETA(epoch): 360s + [Epoch 55][30/33] loss=0.988253 avg=0.875921 VRAM=39.9GiB | 11.2% done | ETA(epoch): 83s + Train loss: 0.871801 (913.6s) ETA: 7444min + Val loss: 0.877833 [t_0.0-0.2=1.0580 t_0.2-0.4=1.0105 t_0.4-0.6=0.8553 t_0.6-0.8=0.7274 t_0.8-1.0=0.6677] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0055 + Deleted old checkpoint: checkpoint_epoch_0052 +[MEM @ epoch 55 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 56/499 (11% done) --- + [Epoch 56][10/33] loss=0.795779 avg=0.844601 VRAM=39.9GiB | 11.3% done | ETA(epoch): 637s + [Epoch 56][20/33] loss=0.910673 avg=0.854957 VRAM=39.6GiB | 11.3% done | ETA(epoch): 360s + [Epoch 56][30/33] loss=0.886852 avg=0.856839 VRAM=39.9GiB | 11.4% done | ETA(epoch): 83s + Train loss: 0.857719 (913.8s) ETA: 7424min + Val loss: 0.867379 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9693 t_0.4-0.6=0.8577 t_0.6-0.8=0.7277 t_0.8-1.0=0.6826] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0056 + Deleted old checkpoint: checkpoint_epoch_0053 +[MEM @ epoch 56 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 57/499 (11% done) --- + [Epoch 57][10/33] loss=0.804697 avg=0.835757 VRAM=39.9GiB | 11.5% done | ETA(epoch): 636s + [Epoch 57][20/33] loss=0.871666 avg=0.840191 VRAM=39.6GiB | 11.5% done | ETA(epoch): 360s + [Epoch 57][30/33] loss=0.847658 avg=0.843692 VRAM=39.9GiB | 11.6% done | ETA(epoch): 83s + Train loss: 0.842015 (913.8s) ETA: 7404min + Val loss: 0.832122 [t_0.0-0.2=1.0805 t_0.2-0.4=0.9612 t_0.4-0.6=0.8587 t_0.6-0.8=0.7318 t_0.8-1.0=0.6551] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0057 (BEST) + Deleted old checkpoint: checkpoint_epoch_0022 + Deleted old checkpoint: checkpoint_epoch_0054 +[MEM @ epoch 57 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 58/499 (12% done) --- + [Epoch 58][10/33] loss=0.867259 avg=0.871195 VRAM=39.9GiB | 11.7% done | ETA(epoch): 637s + [Epoch 58][20/33] loss=0.838003 avg=0.880212 VRAM=39.6GiB | 11.7% done | ETA(epoch): 360s + [Epoch 58][30/33] loss=0.884578 avg=0.872113 VRAM=39.9GiB | 11.8% done | ETA(epoch): 83s + Train loss: 0.870794 (913.7s) ETA: 7384min + Val loss: 0.868584 [t_0.0-0.2=1.0823 t_0.2-0.4=0.9918 t_0.4-0.6=0.8567 t_0.6-0.8=0.7154 t_0.8-1.0=0.6603] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0058 + Deleted old checkpoint: checkpoint_epoch_0055 +[MEM @ epoch 58 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 59/499 (12% done) --- + [Epoch 59][10/33] loss=0.856088 avg=0.874027 VRAM=39.9GiB | 11.9% done | ETA(epoch): 637s + [Epoch 59][20/33] loss=0.906061 avg=0.863212 VRAM=39.6GiB | 11.9% done | ETA(epoch): 360s + [Epoch 59][30/33] loss=0.890306 avg=0.862276 VRAM=39.9GiB | 12.0% done | ETA(epoch): 83s + Train loss: 0.863438 (913.5s) ETA: 7364min + Val loss: 0.836046 [t_0.0-0.2=1.0722 t_0.2-0.4=0.9592 t_0.4-0.6=0.8629 t_0.6-0.8=0.7296 t_0.8-1.0=0.6917] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0059 + Deleted old checkpoint: checkpoint_epoch_0056 +[MEM @ epoch 59 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 60/499 (12% done) --- + [Epoch 60][10/33] loss=0.822375 avg=0.847313 VRAM=39.9GiB | 12.1% done | ETA(epoch): 637s + [Epoch 60][20/33] loss=0.827816 avg=0.838102 VRAM=39.6GiB | 12.1% done | ETA(epoch): 360s + [Epoch 60][30/33] loss=0.952812 avg=0.857500 VRAM=39.9GiB | 12.2% done | ETA(epoch): 83s + Train loss: 0.859058 (914.5s) ETA: 7344min + Val loss: 0.865808 [t_0.0-0.2=1.0913 t_0.2-0.4=1.0092 t_0.4-0.6=0.8372 t_0.6-0.8=0.6956 t_0.8-1.0=0.6802] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0060 +[MEM @ epoch 60 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 61/499 (12% done) --- + [Epoch 61][10/33] loss=0.992221 avg=0.875620 VRAM=39.9GiB | 12.3% done | ETA(epoch): 637s + [Epoch 61][20/33] loss=0.797080 avg=0.864964 VRAM=39.6GiB | 12.3% done | ETA(epoch): 360s + [Epoch 61][30/33] loss=0.925090 avg=0.868151 VRAM=39.9GiB | 12.4% done | ETA(epoch): 83s + Train loss: 0.863724 (914.1s) ETA: 7325min + Val loss: 0.825794 [t_0.0-0.2=1.0841 t_0.2-0.4=0.9538 t_0.4-0.6=0.8595 t_0.6-0.8=0.7143 t_0.8-1.0=0.6742] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0061 (BEST) + Deleted old checkpoint: checkpoint_epoch_0057 + Deleted old checkpoint: checkpoint_epoch_0058 +[MEM @ epoch 61 end] RAM: 15.1/188.4 GiB (8.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 62/499 (12% done) --- + [Epoch 62][10/33] loss=0.844101 avg=0.848001 VRAM=39.9GiB | 12.5% done | ETA(epoch): 637s + [Epoch 62][20/33] loss=0.925305 avg=0.855065 VRAM=39.6GiB | 12.5% done | ETA(epoch): 360s + [Epoch 62][30/33] loss=0.840573 avg=0.858750 VRAM=39.9GiB | 12.6% done | ETA(epoch): 83s + Train loss: 0.862626 (914.2s) ETA: 7306min + Val loss: 0.864788 [t_0.0-0.2=1.0647 t_0.2-0.4=0.9810 t_0.4-0.6=0.8705 t_0.6-0.8=0.7088 t_0.8-1.0=0.6598] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0062 + Deleted old checkpoint: checkpoint_epoch_0059 +[MEM @ epoch 62 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 63/499 (13% done) --- + [Epoch 63][10/33] loss=0.922410 avg=0.886742 VRAM=39.9GiB | 12.7% done | ETA(epoch): 637s + [Epoch 63][20/33] loss=0.896967 avg=0.886877 VRAM=39.6GiB | 12.7% done | ETA(epoch): 360s + [Epoch 63][30/33] loss=0.821960 avg=0.873666 VRAM=39.9GiB | 12.8% done | ETA(epoch): 83s + Train loss: 0.871918 (914.8s) ETA: 7286min + Val loss: 0.871906 [t_0.0-0.2=1.0828 t_0.2-0.4=0.9851 t_0.4-0.6=0.9109 t_0.6-0.8=0.7278 t_0.8-1.0=0.6619] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0063 + Deleted old checkpoint: checkpoint_epoch_0060 +[MEM @ epoch 63 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 64/499 (13% done) --- + [Epoch 64][10/33] loss=0.888220 avg=0.863005 VRAM=39.9GiB | 12.9% done | ETA(epoch): 637s + [Epoch 64][20/33] loss=0.991091 avg=0.878549 VRAM=39.6GiB | 12.9% done | ETA(epoch): 361s + [Epoch 64][30/33] loss=0.841060 avg=0.869037 VRAM=39.9GiB | 13.0% done | ETA(epoch): 83s + Train loss: 0.867798 (915.3s) ETA: 7267min + Val loss: 0.849450 [t_0.0-0.2=1.0674 t_0.2-0.4=1.0140 t_0.4-0.6=0.8582 t_0.6-0.8=0.7170 t_0.8-1.0=0.6454] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0064 +[MEM @ epoch 64 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 65/499 (13% done) --- + [Epoch 65][10/33] loss=0.869027 avg=0.849651 VRAM=39.9GiB | 13.1% done | ETA(epoch): 637s + [Epoch 65][20/33] loss=0.826356 avg=0.854716 VRAM=39.6GiB | 13.1% done | ETA(epoch): 360s + [Epoch 65][30/33] loss=0.858557 avg=0.853016 VRAM=39.9GiB | 13.2% done | ETA(epoch): 83s + Train loss: 0.853899 (915.0s) ETA: 7248min + Val loss: 0.856973 [t_0.0-0.2=1.0880 t_0.2-0.4=0.9848 t_0.4-0.6=0.8187 t_0.6-0.8=0.7209 t_0.8-1.0=0.6662] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0065 + Deleted old checkpoint: checkpoint_epoch_0062 +[MEM @ epoch 65 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 66/499 (13% done) --- + [Epoch 66][10/33] loss=0.880710 avg=0.841337 VRAM=39.9GiB | 13.3% done | ETA(epoch): 638s + [Epoch 66][20/33] loss=0.824028 avg=0.851651 VRAM=39.6GiB | 13.3% done | ETA(epoch): 360s + [Epoch 66][30/33] loss=0.799279 avg=0.861386 VRAM=39.9GiB | 13.4% done | ETA(epoch): 83s + Train loss: 0.863393 (914.3s) ETA: 7229min + Val loss: 0.845727 [t_0.0-0.2=1.0670 t_0.2-0.4=0.9840 t_0.4-0.6=0.8636 t_0.6-0.8=0.7082 t_0.8-1.0=0.6512] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0066 + Deleted old checkpoint: checkpoint_epoch_0063 +[MEM @ epoch 66 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 67/499 (13% done) --- + [Epoch 67][10/33] loss=0.824841 avg=0.843833 VRAM=39.9GiB | 13.5% done | ETA(epoch): 637s + [Epoch 67][20/33] loss=0.932478 avg=0.859197 VRAM=39.6GiB | 13.5% done | ETA(epoch): 360s + [Epoch 67][30/33] loss=0.897746 avg=0.867286 VRAM=39.9GiB | 13.6% done | ETA(epoch): 83s + Train loss: 0.869814 (913.9s) ETA: 7210min + Val loss: 0.890620 [t_0.0-0.2=1.0874 t_0.2-0.4=0.9979 t_0.4-0.6=0.8276 t_0.6-0.8=0.7328 t_0.8-1.0=0.6636] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0067 + Deleted old checkpoint: checkpoint_epoch_0064 +[MEM @ epoch 67 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 68/499 (14% done) --- + [Epoch 68][10/33] loss=0.884246 avg=0.884270 VRAM=39.9GiB | 13.7% done | ETA(epoch): 637s + [Epoch 68][20/33] loss=0.797207 avg=0.878014 VRAM=39.6GiB | 13.7% done | ETA(epoch): 360s + [Epoch 68][30/33] loss=0.762872 avg=0.865565 VRAM=39.9GiB | 13.8% done | ETA(epoch): 83s + Train loss: 0.861500 (913.7s) ETA: 7191min + Val loss: 0.848965 [t_0.0-0.2=1.0733 t_0.2-0.4=0.9621 t_0.4-0.6=0.8608 t_0.6-0.8=0.7082 t_0.8-1.0=0.6753] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0068 + Deleted old checkpoint: checkpoint_epoch_0065 +[MEM @ epoch 68 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 69/499 (14% done) --- + [Epoch 69][10/33] loss=0.819445 avg=0.890005 VRAM=39.9GiB | 13.9% done | ETA(epoch): 637s + [Epoch 69][20/33] loss=0.843246 avg=0.870655 VRAM=39.6GiB | 13.9% done | ETA(epoch): 360s + [Epoch 69][30/33] loss=0.884977 avg=0.866836 VRAM=39.9GiB | 14.0% done | ETA(epoch): 83s + Train loss: 0.861976 (914.0s) ETA: 7173min + Val loss: 0.855137 [t_0.0-0.2=1.0783 t_0.2-0.4=1.0000 t_0.4-0.6=0.8414 t_0.6-0.8=0.7149 t_0.8-1.0=0.6709] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0069 + Deleted old checkpoint: checkpoint_epoch_0066 +[MEM @ epoch 69 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 70/499 (14% done) --- + [Epoch 70][10/33] loss=0.806354 avg=0.861926 VRAM=39.9GiB | 14.1% done | ETA(epoch): 636s + [Epoch 70][20/33] loss=0.985053 avg=0.856991 VRAM=39.6GiB | 14.1% done | ETA(epoch): 360s + [Epoch 70][30/33] loss=0.852513 avg=0.860359 VRAM=39.9GiB | 14.2% done | ETA(epoch): 83s + Train loss: 0.857942 (913.4s) ETA: 7154min + Val loss: 0.839677 [t_0.0-0.2=1.0586 t_0.2-0.4=0.9738 t_0.4-0.6=0.8456 t_0.6-0.8=0.7351 t_0.8-1.0=0.6671] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0070 + Deleted old checkpoint: checkpoint_epoch_0067 +[MEM @ epoch 70 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 71/499 (14% done) --- + [Epoch 71][10/33] loss=0.881777 avg=0.863610 VRAM=39.9GiB | 14.3% done | ETA(epoch): 637s + [Epoch 71][20/33] loss=0.958197 avg=0.861267 VRAM=39.6GiB | 14.3% done | ETA(epoch): 360s + [Epoch 71][30/33] loss=0.870033 avg=0.866726 VRAM=39.9GiB | 14.4% done | ETA(epoch): 83s + Train loss: 0.869343 (913.8s) ETA: 7135min + Val loss: 0.855275 [t_0.0-0.2=1.0746 t_0.2-0.4=1.0037 t_0.4-0.6=0.8594 t_0.6-0.8=0.7245 t_0.8-1.0=0.6504] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0071 + Deleted old checkpoint: checkpoint_epoch_0068 +[MEM @ epoch 71 end] RAM: 15.6/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 72/499 (14% done) --- + [Epoch 72][10/33] loss=0.829622 avg=0.844585 VRAM=39.9GiB | 14.5% done | ETA(epoch): 636s + [Epoch 72][20/33] loss=0.916532 avg=0.862606 VRAM=39.6GiB | 14.5% done | ETA(epoch): 360s + [Epoch 72][30/33] loss=0.939283 avg=0.862958 VRAM=39.9GiB | 14.6% done | ETA(epoch): 83s + Train loss: 0.862224 (913.9s) ETA: 7116min + Val loss: 0.871386 [t_0.0-0.2=1.0682 t_0.2-0.4=0.9994 t_0.4-0.6=0.8418 t_0.6-0.8=0.7411 t_0.8-1.0=0.6647] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0072 + Deleted old checkpoint: checkpoint_epoch_0069 +[MEM @ epoch 72 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 73/499 (15% done) --- + [Epoch 73][10/33] loss=0.850469 avg=0.856360 VRAM=39.9GiB | 14.7% done | ETA(epoch): 637s + [Epoch 73][20/33] loss=0.798851 avg=0.852137 VRAM=39.6GiB | 14.7% done | ETA(epoch): 360s + [Epoch 73][30/33] loss=0.759467 avg=0.862919 VRAM=39.9GiB | 14.8% done | ETA(epoch): 83s + Train loss: 0.865771 (914.1s) ETA: 7098min + Val loss: 0.833843 [t_0.0-0.2=1.0546 t_0.2-0.4=0.9711 t_0.4-0.6=0.8518 t_0.6-0.8=0.7192 t_0.8-1.0=0.6604] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0073 + Deleted old checkpoint: checkpoint_epoch_0070 +[MEM @ epoch 73 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 74/499 (15% done) --- + [Epoch 74][10/33] loss=0.876902 avg=0.862130 VRAM=39.9GiB | 14.9% done | ETA(epoch): 637s + [Epoch 74][20/33] loss=0.804975 avg=0.873933 VRAM=39.6GiB | 14.9% done | ETA(epoch): 360s + [Epoch 74][30/33] loss=0.865411 avg=0.876053 VRAM=39.9GiB | 15.0% done | ETA(epoch): 83s + Train loss: 0.872715 (914.2s) ETA: 7079min + Val loss: 0.865397 [t_0.0-0.2=1.0925 t_0.2-0.4=1.0090 t_0.4-0.6=0.8400 t_0.6-0.8=0.7003 t_0.8-1.0=0.6637] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0074 + Deleted old checkpoint: checkpoint_epoch_0071 +[MEM @ epoch 74 end] RAM: 15.5/188.4 GiB (8.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 75/499 (15% done) --- + [Epoch 75][10/33] loss=0.839990 avg=0.851203 VRAM=39.9GiB | 15.1% done | ETA(epoch): 637s + [Epoch 75][20/33] loss=0.895381 avg=0.860198 VRAM=39.6GiB | 15.1% done | ETA(epoch): 360s + [Epoch 75][30/33] loss=0.874374 avg=0.864428 VRAM=39.9GiB | 15.2% done | ETA(epoch): 83s + Train loss: 0.866631 (914.3s) ETA: 7061min + Val loss: 0.873265 [t_0.0-0.2=1.0658 t_0.2-0.4=0.9873 t_0.4-0.6=0.8598 t_0.6-0.8=0.7065 t_0.8-1.0=0.6671] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0075 + Deleted old checkpoint: checkpoint_epoch_0072 +[MEM @ epoch 75 end] RAM: 15.7/188.4 GiB (8.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 76/499 (15% done) --- + [Epoch 76][10/33] loss=0.961464 avg=0.846569 VRAM=39.9GiB | 15.3% done | ETA(epoch): 637s + [Epoch 76][20/33] loss=0.849275 avg=0.868302 VRAM=39.6GiB | 15.3% done | ETA(epoch): 360s + [Epoch 76][30/33] loss=0.836520 avg=0.865466 VRAM=39.9GiB | 15.4% done | ETA(epoch): 83s + Train loss: 0.868304 (914.2s) ETA: 7043min + Val loss: 0.856831 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9955 t_0.4-0.6=0.8148 t_0.6-0.8=0.7389 t_0.8-1.0=0.6624] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0076 + Deleted old checkpoint: checkpoint_epoch_0073 +[MEM @ epoch 76 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 77/499 (15% done) --- + [Epoch 77][10/33] loss=0.883639 avg=0.863116 VRAM=39.9GiB | 15.5% done | ETA(epoch): 637s + [Epoch 77][20/33] loss=0.894587 avg=0.869585 VRAM=39.6GiB | 15.5% done | ETA(epoch): 360s + [Epoch 77][30/33] loss=0.862202 avg=0.867090 VRAM=39.9GiB | 15.6% done | ETA(epoch): 83s + Train loss: 0.866679 (914.5s) ETA: 7024min + Val loss: 0.858360 [t_0.0-0.2=1.0746 t_0.2-0.4=1.0267 t_0.4-0.6=0.8220 t_0.6-0.8=0.6969 t_0.8-1.0=0.6517] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0077 + Deleted old checkpoint: checkpoint_epoch_0074 +[MEM @ epoch 77 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 78/499 (16% done) --- + [Epoch 78][10/33] loss=0.851706 avg=0.854052 VRAM=39.9GiB | 15.7% done | ETA(epoch): 637s + [Epoch 78][20/33] loss=0.894540 avg=0.854958 VRAM=39.6GiB | 15.7% done | ETA(epoch): 360s + [Epoch 78][30/33] loss=0.812609 avg=0.855624 VRAM=39.9GiB | 15.8% done | ETA(epoch): 83s + Train loss: 0.859115 (914.5s) ETA: 7006min + Val loss: 0.842148 [t_0.0-0.2=1.0822 t_0.2-0.4=0.9753 t_0.4-0.6=0.8371 t_0.6-0.8=0.7191 t_0.8-1.0=0.6754] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0078 + Deleted old checkpoint: checkpoint_epoch_0075 +[MEM @ epoch 78 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 79/499 (16% done) --- + [Epoch 79][10/33] loss=0.909902 avg=0.855451 VRAM=39.9GiB | 15.9% done | ETA(epoch): 637s + [Epoch 79][20/33] loss=0.933228 avg=0.851484 VRAM=39.6GiB | 15.9% done | ETA(epoch): 360s + [Epoch 79][30/33] loss=0.825254 avg=0.851059 VRAM=39.9GiB | 16.0% done | ETA(epoch): 83s + Train loss: 0.850572 (914.4s) ETA: 6988min + Val loss: 0.815813 [t_0.0-0.2=1.0818 t_0.2-0.4=0.9799 t_0.4-0.6=0.8615 t_0.6-0.8=0.7116 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0079 (BEST) + Deleted old checkpoint: checkpoint_epoch_0061 + Deleted old checkpoint: checkpoint_epoch_0076 +[MEM @ epoch 79 end] RAM: 16.7/188.4 GiB (8.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 80/499 (16% done) --- + [Epoch 80][10/33] loss=0.853555 avg=0.866155 VRAM=39.9GiB | 16.1% done | ETA(epoch): 637s + [Epoch 80][20/33] loss=0.922946 avg=0.855995 VRAM=39.6GiB | 16.1% done | ETA(epoch): 360s + [Epoch 80][30/33] loss=0.835916 avg=0.859804 VRAM=39.9GiB | 16.2% done | ETA(epoch): 83s + Train loss: 0.859261 (914.1s) ETA: 6970min + Val loss: 0.831996 [t_0.0-0.2=1.0832 t_0.2-0.4=0.9788 t_0.4-0.6=0.8526 t_0.6-0.8=0.7245 t_0.8-1.0=0.6522] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0080 + Deleted old checkpoint: checkpoint_epoch_0077 +[MEM @ epoch 80 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 81/499 (16% done) --- + [Epoch 81][10/33] loss=0.923219 avg=0.877744 VRAM=39.9GiB | 16.3% done | ETA(epoch): 637s + [Epoch 81][20/33] loss=0.878037 avg=0.866585 VRAM=39.6GiB | 16.3% done | ETA(epoch): 360s + [Epoch 81][30/33] loss=0.898546 avg=0.860115 VRAM=39.9GiB | 16.4% done | ETA(epoch): 83s + Train loss: 0.856431 (914.3s) ETA: 6952min + Val loss: 0.834373 [t_0.0-0.2=1.0582 t_0.2-0.4=1.0310 t_0.4-0.6=0.8389 t_0.6-0.8=0.7261 t_0.8-1.0=0.6629] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0081 + Deleted old checkpoint: checkpoint_epoch_0078 +[MEM @ epoch 81 end] RAM: 16.7/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 82/499 (16% done) --- + [Epoch 82][10/33] loss=0.879373 avg=0.856517 VRAM=39.9GiB | 16.5% done | ETA(epoch): 637s + [Epoch 82][20/33] loss=0.970330 avg=0.872380 VRAM=39.6GiB | 16.5% done | ETA(epoch): 360s + [Epoch 82][30/33] loss=0.833022 avg=0.862372 VRAM=39.9GiB | 16.6% done | ETA(epoch): 83s + Train loss: 0.868423 (914.0s) ETA: 6934min + Val loss: 0.875715 [t_0.0-0.2=1.0870 t_0.2-0.4=0.9950 t_0.4-0.6=0.8928 t_0.6-0.8=0.7007 t_0.8-1.0=0.6569] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0082 +[MEM @ epoch 82 end] RAM: 16.6/188.4 GiB (8.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 83/499 (17% done) --- + [Epoch 83][10/33] loss=0.785750 avg=0.821799 VRAM=39.9GiB | 16.7% done | ETA(epoch): 637s + [Epoch 83][20/33] loss=0.832112 avg=0.833163 VRAM=39.6GiB | 16.7% done | ETA(epoch): 360s + [Epoch 83][30/33] loss=0.897478 avg=0.844012 VRAM=39.9GiB | 16.8% done | ETA(epoch): 83s + Train loss: 0.844775 (913.7s) ETA: 6915min + Val loss: 0.893376 [t_0.0-0.2=1.0743 t_0.2-0.4=1.0011 t_0.4-0.6=0.8226 t_0.6-0.8=0.7316 t_0.8-1.0=0.6604] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0083 + Deleted old checkpoint: checkpoint_epoch_0080 +[MEM @ epoch 83 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 84/499 (17% done) --- + [Epoch 84][10/33] loss=0.822119 avg=0.847493 VRAM=39.9GiB | 16.9% done | ETA(epoch): 636s + [Epoch 84][20/33] loss=0.909156 avg=0.860402 VRAM=39.6GiB | 16.9% done | ETA(epoch): 360s + [Epoch 84][30/33] loss=0.914587 avg=0.857364 VRAM=39.9GiB | 17.0% done | ETA(epoch): 83s + Train loss: 0.855103 (912.8s) ETA: 6897min + Val loss: 0.851875 [t_0.0-0.2=1.0720 t_0.2-0.4=1.0157 t_0.4-0.6=0.7919 t_0.6-0.8=0.7149 t_0.8-1.0=0.6540] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0084 + Deleted old checkpoint: checkpoint_epoch_0081 +[MEM @ epoch 84 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 85/499 (17% done) --- + [Epoch 85][10/33] loss=0.924068 avg=0.891538 VRAM=39.9GiB | 17.1% done | ETA(epoch): 636s + [Epoch 85][20/33] loss=0.902107 avg=0.873771 VRAM=39.6GiB | 17.1% done | ETA(epoch): 360s + [Epoch 85][30/33] loss=0.881904 avg=0.866339 VRAM=39.9GiB | 17.2% done | ETA(epoch): 83s + Train loss: 0.863746 (913.1s) ETA: 6879min + Val loss: 0.887459 [t_0.0-0.2=1.0833 t_0.2-0.4=0.9964 t_0.4-0.6=0.8587 t_0.6-0.8=0.7259 t_0.8-1.0=0.6457] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0085 + Deleted old checkpoint: checkpoint_epoch_0082 +[MEM @ epoch 85 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 86/499 (17% done) --- + [Epoch 86][10/33] loss=0.814737 avg=0.865634 VRAM=39.9GiB | 17.3% done | ETA(epoch): 636s + [Epoch 86][20/33] loss=0.819941 avg=0.856621 VRAM=39.6GiB | 17.3% done | ETA(epoch): 359s + [Epoch 86][30/33] loss=0.808586 avg=0.859102 VRAM=39.9GiB | 17.4% done | ETA(epoch): 83s + Train loss: 0.862954 (912.7s) ETA: 6861min + Val loss: 0.838212 [t_0.0-0.2=1.0833 t_0.2-0.4=0.9886 t_0.4-0.6=0.8568 t_0.6-0.8=0.7117 t_0.8-1.0=0.6589] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0086 + Deleted old checkpoint: checkpoint_epoch_0083 +[MEM @ epoch 86 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 87/499 (17% done) --- + [Epoch 87][10/33] loss=0.957545 avg=0.879236 VRAM=39.9GiB | 17.5% done | ETA(epoch): 636s + [Epoch 87][20/33] loss=0.842983 avg=0.885033 VRAM=39.6GiB | 17.5% done | ETA(epoch): 360s + [Epoch 87][30/33] loss=0.925694 avg=0.874060 VRAM=39.9GiB | 17.6% done | ETA(epoch): 83s + Train loss: 0.874859 (913.4s) ETA: 6843min + Val loss: 0.842103 [t_0.0-0.2=1.0600 t_0.2-0.4=0.9823 t_0.4-0.6=0.8709 t_0.6-0.8=0.7240 t_0.8-1.0=0.6689] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0087 + Deleted old checkpoint: checkpoint_epoch_0084 +[MEM @ epoch 87 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 88/499 (18% done) --- + [Epoch 88][10/33] loss=0.741387 avg=0.843801 VRAM=39.9GiB | 17.7% done | ETA(epoch): 636s + [Epoch 88][20/33] loss=0.893389 avg=0.853047 VRAM=39.6GiB | 17.7% done | ETA(epoch): 360s + [Epoch 88][30/33] loss=0.893844 avg=0.861550 VRAM=39.9GiB | 17.8% done | ETA(epoch): 83s + Train loss: 0.858828 (913.2s) ETA: 6825min + Val loss: 0.855564 [t_0.0-0.2=1.0703 t_0.2-0.4=1.0076 t_0.4-0.6=0.8196 t_0.6-0.8=0.7126 t_0.8-1.0=0.6679] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0088 + Deleted old checkpoint: checkpoint_epoch_0085 +[MEM @ epoch 88 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 89/499 (18% done) --- + [Epoch 89][10/33] loss=0.904675 avg=0.848525 VRAM=39.9GiB | 17.9% done | ETA(epoch): 636s + [Epoch 89][20/33] loss=0.930109 avg=0.851788 VRAM=39.6GiB | 17.9% done | ETA(epoch): 360s + [Epoch 89][30/33] loss=0.829482 avg=0.853856 VRAM=39.9GiB | 18.0% done | ETA(epoch): 83s + Train loss: 0.854088 (912.9s) ETA: 6807min + Val loss: 0.845974 [t_0.0-0.2=1.0800 t_0.2-0.4=0.9956 t_0.4-0.6=0.8484 t_0.6-0.8=0.7237 t_0.8-1.0=0.6511] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0089 + Deleted old checkpoint: checkpoint_epoch_0086 +[MEM @ epoch 89 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 90/499 (18% done) --- + [Epoch 90][10/33] loss=0.739942 avg=0.840468 VRAM=39.9GiB | 18.1% done | ETA(epoch): 636s + [Epoch 90][20/33] loss=0.870363 avg=0.852716 VRAM=39.6GiB | 18.1% done | ETA(epoch): 360s + [Epoch 90][30/33] loss=0.855298 avg=0.857299 VRAM=39.9GiB | 18.2% done | ETA(epoch): 83s + Train loss: 0.853225 (913.0s) ETA: 6790min + Val loss: 0.873656 [t_0.0-0.2=1.0458 t_0.2-0.4=1.0102 t_0.4-0.6=0.8550 t_0.6-0.8=0.6894 t_0.8-1.0=0.6580] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0090 + Deleted old checkpoint: checkpoint_epoch_0087 +[MEM @ epoch 90 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 91/499 (18% done) --- + [Epoch 91][10/33] loss=0.789873 avg=0.845064 VRAM=39.9GiB | 18.3% done | ETA(epoch): 637s + [Epoch 91][20/33] loss=0.897470 avg=0.848658 VRAM=39.6GiB | 18.3% done | ETA(epoch): 360s + [Epoch 91][30/33] loss=0.772473 avg=0.849761 VRAM=39.9GiB | 18.4% done | ETA(epoch): 83s + Train loss: 0.851746 (913.9s) ETA: 6772min + Val loss: 0.854178 [t_0.0-0.2=1.0775 t_0.2-0.4=0.9749 t_0.4-0.6=0.8432 t_0.6-0.8=0.7193 t_0.8-1.0=0.6620] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0091 + Deleted old checkpoint: checkpoint_epoch_0088 +[MEM @ epoch 91 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 92/499 (18% done) --- + [Epoch 92][10/33] loss=0.813808 avg=0.869136 VRAM=39.9GiB | 18.5% done | ETA(epoch): 637s + [Epoch 92][20/33] loss=0.836122 avg=0.861098 VRAM=39.6GiB | 18.5% done | ETA(epoch): 360s + [Epoch 92][30/33] loss=0.880912 avg=0.865098 VRAM=39.9GiB | 18.6% done | ETA(epoch): 83s + Train loss: 0.868403 (913.7s) ETA: 6754min + Val loss: 0.828631 [t_0.0-0.2=1.0763 t_0.2-0.4=1.0121 t_0.4-0.6=0.8397 t_0.6-0.8=0.7239 t_0.8-1.0=0.6664] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0092 + Deleted old checkpoint: checkpoint_epoch_0089 +[MEM @ epoch 92 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 93/499 (19% done) --- + [Epoch 93][10/33] loss=0.936742 avg=0.849209 VRAM=39.9GiB | 18.7% done | ETA(epoch): 636s + [Epoch 93][20/33] loss=0.876070 avg=0.859642 VRAM=39.6GiB | 18.7% done | ETA(epoch): 360s + [Epoch 93][30/33] loss=0.931709 avg=0.857489 VRAM=39.9GiB | 18.8% done | ETA(epoch): 83s + Train loss: 0.856484 (913.0s) ETA: 6736min + Val loss: 0.850398 [t_0.0-0.2=1.0617 t_0.2-0.4=0.9655 t_0.4-0.6=0.8740 t_0.6-0.8=0.7131 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0093 + Deleted old checkpoint: checkpoint_epoch_0090 +[MEM @ epoch 93 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 94/499 (19% done) --- + [Epoch 94][10/33] loss=0.966770 avg=0.861161 VRAM=39.9GiB | 18.9% done | ETA(epoch): 636s + [Epoch 94][20/33] loss=0.819965 avg=0.865647 VRAM=39.6GiB | 18.9% done | ETA(epoch): 359s + [Epoch 94][30/33] loss=0.884723 avg=0.854347 VRAM=39.9GiB | 19.0% done | ETA(epoch): 83s + Train loss: 0.853189 (912.3s) ETA: 6718min + Val loss: 0.856903 [t_0.0-0.2=1.0799 t_0.2-0.4=1.0249 t_0.4-0.6=0.8578 t_0.6-0.8=0.7166 t_0.8-1.0=0.6681] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0094 + Deleted old checkpoint: checkpoint_epoch_0091 +[MEM @ epoch 94 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 95/499 (19% done) --- + [Epoch 95][10/33] loss=0.796453 avg=0.887483 VRAM=39.9GiB | 19.1% done | ETA(epoch): 635s + [Epoch 95][20/33] loss=0.829122 avg=0.859558 VRAM=39.6GiB | 19.1% done | ETA(epoch): 359s + [Epoch 95][30/33] loss=0.921454 avg=0.865807 VRAM=39.9GiB | 19.2% done | ETA(epoch): 83s + Train loss: 0.868353 (912.7s) ETA: 6701min + Val loss: 0.863059 [t_0.0-0.2=1.0708 t_0.2-0.4=1.0115 t_0.4-0.6=0.8437 t_0.6-0.8=0.7149 t_0.8-1.0=0.6564] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0095 + Deleted old checkpoint: checkpoint_epoch_0092 +[MEM @ epoch 95 end] RAM: 17.0/188.4 GiB (9.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 96/499 (19% done) --- + [Epoch 96][10/33] loss=0.918530 avg=0.890506 VRAM=39.9GiB | 19.3% done | ETA(epoch): 637s + [Epoch 96][20/33] loss=0.845849 avg=0.875707 VRAM=39.6GiB | 19.3% done | ETA(epoch): 360s + [Epoch 96][30/33] loss=0.864313 avg=0.873987 VRAM=39.9GiB | 19.4% done | ETA(epoch): 83s + Train loss: 0.867559 (913.4s) ETA: 6683min + Val loss: 0.851557 [t_0.0-0.2=1.0650 t_0.2-0.4=0.9898 t_0.4-0.6=0.8402 t_0.6-0.8=0.7357 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0096 + Deleted old checkpoint: checkpoint_epoch_0093 +[MEM @ epoch 96 end] RAM: 17.2/188.4 GiB (9.1%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 97/499 (19% done) --- + [Epoch 97][10/33] loss=0.879367 avg=0.846060 VRAM=39.9GiB | 19.5% done | ETA(epoch): 638s + [Epoch 97][20/33] loss=0.886580 avg=0.862048 VRAM=39.6GiB | 19.5% done | ETA(epoch): 360s + [Epoch 97][30/33] loss=0.847570 avg=0.865166 VRAM=39.9GiB | 19.6% done | ETA(epoch): 83s + Train loss: 0.862492 (914.2s) ETA: 6665min + Val loss: 0.842061 [t_0.0-0.2=1.0823 t_0.2-0.4=0.9660 t_0.4-0.6=0.8518 t_0.6-0.8=0.7321 t_0.8-1.0=0.6466] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0097 + Deleted old checkpoint: checkpoint_epoch_0094 +[MEM @ epoch 97 end] RAM: 17.2/188.4 GiB (9.1%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 98/499 (20% done) --- + [Epoch 98][10/33] loss=0.878900 avg=0.878693 VRAM=39.9GiB | 19.7% done | ETA(epoch): 636s + [Epoch 98][20/33] loss=0.881487 avg=0.889901 VRAM=39.6GiB | 19.7% done | ETA(epoch): 360s + [Epoch 98][30/33] loss=0.975824 avg=0.888634 VRAM=39.9GiB | 19.8% done | ETA(epoch): 83s + Train loss: 0.886882 (913.4s) ETA: 6648min + Val loss: 0.850965 [t_0.0-0.2=1.0745 t_0.2-0.4=0.9688 t_0.4-0.6=0.8451 t_0.6-0.8=0.7262 t_0.8-1.0=0.6531] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0098 + Deleted old checkpoint: checkpoint_epoch_0095 +[MEM @ epoch 98 end] RAM: 17.3/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 99/499 (20% done) --- + [Epoch 99][10/33] loss=0.838224 avg=0.828640 VRAM=39.9GiB | 19.9% done | ETA(epoch): 636s + [Epoch 99][20/33] loss=0.896017 avg=0.832576 VRAM=39.6GiB | 19.9% done | ETA(epoch): 360s + [Epoch 99][30/33] loss=0.865180 avg=0.841599 VRAM=39.9GiB | 20.0% done | ETA(epoch): 83s + Train loss: 0.843782 (913.4s) ETA: 6630min + Val loss: 0.881864 [t_0.0-0.2=1.0797 t_0.2-0.4=0.9989 t_0.4-0.6=0.8607 t_0.6-0.8=0.7086 t_0.8-1.0=0.6577] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0099 + Deleted old checkpoint: checkpoint_epoch_0096 +[MEM @ epoch 99 end] RAM: 17.4/188.4 GiB (9.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 100/499 (20% done) --- + [MilestoneVis] train_0 step 3300 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 3300 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 3300 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 3300 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 3300 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 3300 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 3300 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 3300 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 3300 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 3300 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 100][10/33] loss=0.871200 avg=0.870070 VRAM=39.9GiB | 20.1% done | ETA(epoch): 2369s + [Epoch 100][20/33] loss=0.753694 avg=0.850236 VRAM=39.6GiB | 20.1% done | ETA(epoch): 849s + [Epoch 100][30/33] loss=0.764629 avg=0.854493 VRAM=39.9GiB | 20.2% done | ETA(epoch): 158s + Train loss: 0.856928 (1666.2s) ETA: 6662min + Val loss: 0.847418 [t_0.0-0.2=1.0661 t_0.2-0.4=0.9937 t_0.4-0.6=0.8313 t_0.6-0.8=0.7379 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0100 + Deleted old checkpoint: checkpoint_epoch_0097 +[MEM @ epoch 100 end] RAM: 17.9/188.4 GiB (9.5%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 101/499 (20% done) --- + [Epoch 101][10/33] loss=0.898299 avg=0.874088 VRAM=39.9GiB | 20.3% done | ETA(epoch): 636s + [Epoch 101][20/33] loss=0.841536 avg=0.865980 VRAM=39.6GiB | 20.3% done | ETA(epoch): 360s + [Epoch 101][30/33] loss=0.825620 avg=0.878048 VRAM=39.9GiB | 20.4% done | ETA(epoch): 83s + Train loss: 0.876598 (913.5s) ETA: 6644min + Val loss: 0.858945 [t_0.0-0.2=1.0725 t_0.2-0.4=1.0203 t_0.4-0.6=0.7999 t_0.6-0.8=0.7023 t_0.8-1.0=0.6631] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0101 + Deleted old checkpoint: checkpoint_epoch_0098 +[MEM @ epoch 101 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 102/499 (20% done) --- + [Epoch 102][10/33] loss=0.840769 avg=0.844765 VRAM=39.9GiB | 20.5% done | ETA(epoch): 637s + [Epoch 102][20/33] loss=0.875137 avg=0.864639 VRAM=39.6GiB | 20.5% done | ETA(epoch): 360s + [Epoch 102][30/33] loss=0.784545 avg=0.855079 VRAM=39.9GiB | 20.6% done | ETA(epoch): 83s + Train loss: 0.854421 (913.9s) ETA: 6626min + Val loss: 0.831981 [t_0.0-0.2=1.0693 t_0.2-0.4=0.9743 t_0.4-0.6=0.8474 t_0.6-0.8=0.7104 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0102 + Deleted old checkpoint: checkpoint_epoch_0099 +[MEM @ epoch 102 end] RAM: 17.4/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 103/499 (21% done) --- + [Epoch 103][10/33] loss=0.852770 avg=0.864188 VRAM=39.9GiB | 20.7% done | ETA(epoch): 636s + [Epoch 103][20/33] loss=0.863920 avg=0.865755 VRAM=39.6GiB | 20.7% done | ETA(epoch): 360s + [Epoch 103][30/33] loss=0.941928 avg=0.866380 VRAM=39.9GiB | 20.8% done | ETA(epoch): 83s + Train loss: 0.863058 (913.2s) ETA: 6608min + Val loss: 0.870796 [t_0.0-0.2=1.0734 t_0.2-0.4=1.0036 t_0.4-0.6=0.8414 t_0.6-0.8=0.6957 t_0.8-1.0=0.6567] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0103 + Deleted old checkpoint: checkpoint_epoch_0100 +[MEM @ epoch 103 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 104/499 (21% done) --- + [Epoch 104][10/33] loss=0.895286 avg=0.873795 VRAM=39.9GiB | 20.9% done | ETA(epoch): 637s + [Epoch 104][20/33] loss=0.783691 avg=0.863077 VRAM=39.6GiB | 20.9% done | ETA(epoch): 360s + [Epoch 104][30/33] loss=0.950283 avg=0.857225 VRAM=39.9GiB | 21.0% done | ETA(epoch): 83s + Train loss: 0.860085 (913.5s) ETA: 6590min + Val loss: 0.862211 [t_0.0-0.2=1.0594 t_0.2-0.4=0.9980 t_0.4-0.6=0.8548 t_0.6-0.8=0.7117 t_0.8-1.0=0.6594] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0104 + Deleted old checkpoint: checkpoint_epoch_0101 +[MEM @ epoch 104 end] RAM: 17.5/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 105/499 (21% done) --- + [Epoch 105][10/33] loss=0.861134 avg=0.857884 VRAM=39.9GiB | 21.1% done | ETA(epoch): 636s + [Epoch 105][20/33] loss=0.791277 avg=0.870308 VRAM=39.6GiB | 21.1% done | ETA(epoch): 360s + [Epoch 105][30/33] loss=0.934408 avg=0.869255 VRAM=39.9GiB | 21.2% done | ETA(epoch): 83s + Train loss: 0.870327 (913.0s) ETA: 6572min + Val loss: 0.865428 [t_0.0-0.2=1.0716 t_0.2-0.4=0.9939 t_0.4-0.6=0.8224 t_0.6-0.8=0.7226 t_0.8-1.0=0.6625] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0105 + Deleted old checkpoint: checkpoint_epoch_0102 +[MEM @ epoch 105 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 106/499 (21% done) --- + [Epoch 106][10/33] loss=0.862740 avg=0.851507 VRAM=39.9GiB | 21.3% done | ETA(epoch): 637s + [Epoch 106][20/33] loss=0.890128 avg=0.850603 VRAM=39.6GiB | 21.3% done | ETA(epoch): 360s + [Epoch 106][30/33] loss=0.889292 avg=0.847228 VRAM=39.9GiB | 21.4% done | ETA(epoch): 83s + Train loss: 0.850700 (913.4s) ETA: 6554min + Val loss: 0.843132 [t_0.0-0.2=1.0618 t_0.2-0.4=0.9816 t_0.4-0.6=0.8452 t_0.6-0.8=0.7216 t_0.8-1.0=0.6556] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0106 + Deleted old checkpoint: checkpoint_epoch_0103 +[MEM @ epoch 106 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 107/499 (21% done) --- + [Epoch 107][10/33] loss=0.834193 avg=0.865757 VRAM=39.9GiB | 21.5% done | ETA(epoch): 636s + [Epoch 107][20/33] loss=0.852098 avg=0.869822 VRAM=39.6GiB | 21.5% done | ETA(epoch): 360s + [Epoch 107][30/33] loss=0.849137 avg=0.859507 VRAM=39.9GiB | 21.6% done | ETA(epoch): 83s + Train loss: 0.861582 (913.3s) ETA: 6536min + Val loss: 0.837612 [t_0.0-0.2=1.0836 t_0.2-0.4=1.0220 t_0.4-0.6=0.8356 t_0.6-0.8=0.6884 t_0.8-1.0=0.6563] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0107 + Deleted old checkpoint: checkpoint_epoch_0104 +[MEM @ epoch 107 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 108/499 (22% done) --- + [Epoch 108][10/33] loss=0.880051 avg=0.867363 VRAM=39.9GiB | 21.7% done | ETA(epoch): 636s + [Epoch 108][20/33] loss=0.936281 avg=0.868337 VRAM=39.6GiB | 21.7% done | ETA(epoch): 360s + [Epoch 108][30/33] loss=0.772904 avg=0.860929 VRAM=39.9GiB | 21.8% done | ETA(epoch): 83s + Train loss: 0.866205 (913.3s) ETA: 6518min + Val loss: 0.862674 [t_0.0-0.2=1.0730 t_0.2-0.4=0.9937 t_0.4-0.6=0.8660 t_0.6-0.8=0.7095 t_0.8-1.0=0.6416] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0108 + Deleted old checkpoint: checkpoint_epoch_0105 +[MEM @ epoch 108 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 109/499 (22% done) --- + [Epoch 109][10/33] loss=0.868762 avg=0.866920 VRAM=39.9GiB | 21.9% done | ETA(epoch): 637s + [Epoch 109][20/33] loss=0.825163 avg=0.870593 VRAM=39.6GiB | 21.9% done | ETA(epoch): 360s + [Epoch 109][30/33] loss=0.857690 avg=0.863641 VRAM=39.9GiB | 22.0% done | ETA(epoch): 83s + Train loss: 0.861899 (913.3s) ETA: 6500min + Val loss: 0.855663 [t_0.0-0.2=1.0771 t_0.2-0.4=1.0101 t_0.4-0.6=0.8254 t_0.6-0.8=0.6812 t_0.8-1.0=0.6507] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0109 + Deleted old checkpoint: checkpoint_epoch_0106 +[MEM @ epoch 109 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 110/499 (22% done) --- + [Epoch 110][10/33] loss=0.818724 avg=0.847481 VRAM=39.9GiB | 22.1% done | ETA(epoch): 637s + [Epoch 110][20/33] loss=0.840641 avg=0.850893 VRAM=39.6GiB | 22.1% done | ETA(epoch): 360s + [Epoch 110][30/33] loss=0.762980 avg=0.849353 VRAM=39.9GiB | 22.2% done | ETA(epoch): 83s + Train loss: 0.849109 (913.2s) ETA: 6482min + Val loss: 0.800155 [t_0.0-0.2=1.0723 t_0.2-0.4=0.9956 t_0.4-0.6=0.8485 t_0.6-0.8=0.7033 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0110 (BEST) + Deleted old checkpoint: checkpoint_epoch_0079 + Deleted old checkpoint: checkpoint_epoch_0107 +[MEM @ epoch 110 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 111/499 (22% done) --- + [Epoch 111][10/33] loss=0.954246 avg=0.912876 VRAM=39.9GiB | 22.3% done | ETA(epoch): 637s + [Epoch 111][20/33] loss=0.736166 avg=0.870126 VRAM=39.6GiB | 22.3% done | ETA(epoch): 360s + [Epoch 111][30/33] loss=0.765180 avg=0.858409 VRAM=39.9GiB | 22.4% done | ETA(epoch): 83s + Train loss: 0.857193 (914.0s) ETA: 6464min + Val loss: 0.853525 [t_0.0-0.2=1.0749 t_0.2-0.4=0.9816 t_0.4-0.6=0.8354 t_0.6-0.8=0.7350 t_0.8-1.0=0.6528] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0111 + Deleted old checkpoint: checkpoint_epoch_0108 +[MEM @ epoch 111 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 112/499 (22% done) --- + [Epoch 112][10/33] loss=0.825397 avg=0.838515 VRAM=39.9GiB | 22.5% done | ETA(epoch): 637s + [Epoch 112][20/33] loss=0.797314 avg=0.842303 VRAM=39.6GiB | 22.5% done | ETA(epoch): 360s + [Epoch 112][30/33] loss=0.914525 avg=0.844903 VRAM=39.9GiB | 22.6% done | ETA(epoch): 83s + Train loss: 0.848493 (914.3s) ETA: 6447min + Val loss: 0.841253 [t_0.0-0.2=1.0501 t_0.2-0.4=1.0094 t_0.4-0.6=0.8155 t_0.6-0.8=0.7278 t_0.8-1.0=0.6642] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0112 + Deleted old checkpoint: checkpoint_epoch_0109 +[MEM @ epoch 112 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 113/499 (23% done) --- + [Epoch 113][10/33] loss=0.870819 avg=0.853294 VRAM=39.9GiB | 22.7% done | ETA(epoch): 637s + [Epoch 113][20/33] loss=0.817502 avg=0.860762 VRAM=39.6GiB | 22.7% done | ETA(epoch): 360s + [Epoch 113][30/33] loss=0.894072 avg=0.856588 VRAM=39.9GiB | 22.8% done | ETA(epoch): 83s + Train loss: 0.853996 (914.3s) ETA: 6429min + Val loss: 0.860767 [t_0.0-0.2=1.0798 t_0.2-0.4=1.0013 t_0.4-0.6=0.8663 t_0.6-0.8=0.7051 t_0.8-1.0=0.6426] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0113 +[MEM @ epoch 113 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 114/499 (23% done) --- + [Epoch 114][10/33] loss=0.799565 avg=0.854629 VRAM=39.9GiB | 22.9% done | ETA(epoch): 637s + [Epoch 114][20/33] loss=0.872581 avg=0.846402 VRAM=39.6GiB | 22.9% done | ETA(epoch): 360s + [Epoch 114][30/33] loss=0.832155 avg=0.850265 VRAM=39.9GiB | 23.0% done | ETA(epoch): 83s + Train loss: 0.853391 (914.3s) ETA: 6411min + Val loss: 0.830289 [t_0.0-0.2=1.0730 t_0.2-0.4=1.0381 t_0.4-0.6=0.8164 t_0.6-0.8=0.7292 t_0.8-1.0=0.6374] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0114 + Deleted old checkpoint: checkpoint_epoch_0111 +[MEM @ epoch 114 end] RAM: 17.7/188.4 GiB (9.4%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 115/499 (23% done) --- + [Epoch 115][10/33] loss=0.935071 avg=0.837511 VRAM=39.9GiB | 23.1% done | ETA(epoch): 637s + [Epoch 115][20/33] loss=0.876603 avg=0.847352 VRAM=39.6GiB | 23.1% done | ETA(epoch): 360s + [Epoch 115][30/33] loss=0.779114 avg=0.845174 VRAM=39.9GiB | 23.2% done | ETA(epoch): 83s + Train loss: 0.847936 (914.0s) ETA: 6393min + Val loss: 0.870996 [t_0.0-0.2=1.0743 t_0.2-0.4=1.0065 t_0.4-0.6=0.8518 t_0.6-0.8=0.7114 t_0.8-1.0=0.6496] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0115 + Deleted old checkpoint: checkpoint_epoch_0112 +[MEM @ epoch 115 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 116/499 (23% done) --- + [Epoch 116][10/33] loss=0.839608 avg=0.854443 VRAM=39.9GiB | 23.3% done | ETA(epoch): 636s + [Epoch 116][20/33] loss=0.860792 avg=0.860657 VRAM=39.6GiB | 23.3% done | ETA(epoch): 360s + [Epoch 116][30/33] loss=0.877548 avg=0.860900 VRAM=39.9GiB | 23.4% done | ETA(epoch): 83s + Train loss: 0.861374 (913.6s) ETA: 6376min + Val loss: 0.850246 [t_0.0-0.2=1.0691 t_0.2-0.4=1.0046 t_0.4-0.6=0.8239 t_0.6-0.8=0.7044 t_0.8-1.0=0.6648] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0116 + Deleted old checkpoint: checkpoint_epoch_0113 +[MEM @ epoch 116 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 117/499 (23% done) --- + [Epoch 117][10/33] loss=0.950386 avg=0.855137 VRAM=39.9GiB | 23.5% done | ETA(epoch): 637s + [Epoch 117][20/33] loss=0.792341 avg=0.861253 VRAM=39.6GiB | 23.5% done | ETA(epoch): 360s + [Epoch 117][30/33] loss=0.834311 avg=0.862940 VRAM=39.9GiB | 23.6% done | ETA(epoch): 83s + Train loss: 0.857061 (913.9s) ETA: 6358min + Val loss: 0.872169 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9744 t_0.4-0.6=0.8791 t_0.6-0.8=0.7182 t_0.8-1.0=0.6498] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0117 + Deleted old checkpoint: checkpoint_epoch_0114 +[MEM @ epoch 117 end] RAM: 17.6/188.4 GiB (9.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 118/499 (24% done) --- + [Epoch 118][10/33] loss=0.849347 avg=0.846848 VRAM=39.9GiB | 23.7% done | ETA(epoch): 637s + [Epoch 118][20/33] loss=0.848779 avg=0.849441 VRAM=39.6GiB | 23.7% done | ETA(epoch): 360s + [Epoch 118][30/33] loss=0.844975 avg=0.848872 VRAM=39.9GiB | 23.8% done | ETA(epoch): 83s + Train loss: 0.849524 (914.0s) ETA: 6340min + Val loss: 0.852797 [t_0.0-0.2=1.0759 t_0.2-0.4=0.9977 t_0.4-0.6=0.8277 t_0.6-0.8=0.7033 t_0.8-1.0=0.6501] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0118 + Deleted old checkpoint: checkpoint_epoch_0115 +[MEM @ epoch 118 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 119/499 (24% done) --- + [Epoch 119][10/33] loss=0.852227 avg=0.865941 VRAM=39.9GiB | 23.9% done | ETA(epoch): 637s + [Epoch 119][20/33] loss=0.845729 avg=0.854228 VRAM=39.6GiB | 23.9% done | ETA(epoch): 360s + [Epoch 119][30/33] loss=0.836724 avg=0.865291 VRAM=39.9GiB | 24.0% done | ETA(epoch): 83s + Train loss: 0.862206 (914.1s) ETA: 6323min + Val loss: 0.825141 [t_0.0-0.2=1.0596 t_0.2-0.4=0.9918 t_0.4-0.6=0.8514 t_0.6-0.8=0.7117 t_0.8-1.0=0.6474] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0119 + Deleted old checkpoint: checkpoint_epoch_0116 +[MEM @ epoch 119 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 120/499 (24% done) --- + [Epoch 120][10/33] loss=0.908431 avg=0.872655 VRAM=39.9GiB | 24.1% done | ETA(epoch): 637s + [Epoch 120][20/33] loss=0.766765 avg=0.856170 VRAM=39.6GiB | 24.1% done | ETA(epoch): 360s + [Epoch 120][30/33] loss=0.803150 avg=0.857777 VRAM=39.9GiB | 24.2% done | ETA(epoch): 83s + Train loss: 0.855377 (914.0s) ETA: 6305min + Val loss: 0.830963 [t_0.0-0.2=1.0699 t_0.2-0.4=0.9762 t_0.4-0.6=0.8508 t_0.6-0.8=0.7223 t_0.8-1.0=0.6534] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0120 + Deleted old checkpoint: checkpoint_epoch_0117 +[MEM @ epoch 120 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 121/499 (24% done) --- + [Epoch 121][10/33] loss=0.794263 avg=0.872195 VRAM=39.9GiB | 24.3% done | ETA(epoch): 637s + [Epoch 121][20/33] loss=0.854679 avg=0.853024 VRAM=39.6GiB | 24.3% done | ETA(epoch): 360s + [Epoch 121][30/33] loss=0.868884 avg=0.845584 VRAM=39.9GiB | 24.4% done | ETA(epoch): 83s + Train loss: 0.845358 (914.0s) ETA: 6288min + Val loss: 0.866362 [t_0.0-0.2=1.0701 t_0.2-0.4=1.0110 t_0.4-0.6=0.8478 t_0.6-0.8=0.7021 t_0.8-1.0=0.6360] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0121 + Deleted old checkpoint: checkpoint_epoch_0118 +[MEM @ epoch 121 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 122/499 (24% done) --- + [Epoch 122][10/33] loss=0.839699 avg=0.869047 VRAM=39.9GiB | 24.5% done | ETA(epoch): 637s + [Epoch 122][20/33] loss=0.836821 avg=0.851857 VRAM=39.6GiB | 24.5% done | ETA(epoch): 360s + [Epoch 122][30/33] loss=0.779896 avg=0.846601 VRAM=39.9GiB | 24.6% done | ETA(epoch): 83s + Train loss: 0.851395 (914.4s) ETA: 6270min + Val loss: 0.849162 [t_0.0-0.2=1.0653 t_0.2-0.4=1.0062 t_0.4-0.6=0.9141 t_0.6-0.8=0.7070 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0122 + Deleted old checkpoint: checkpoint_epoch_0119 +[MEM @ epoch 122 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 123/499 (25% done) --- + [Epoch 123][10/33] loss=0.764703 avg=0.824778 VRAM=39.9GiB | 24.7% done | ETA(epoch): 637s + [Epoch 123][20/33] loss=0.749072 avg=0.841062 VRAM=39.6GiB | 24.7% done | ETA(epoch): 360s + [Epoch 123][30/33] loss=0.913739 avg=0.852078 VRAM=39.9GiB | 24.8% done | ETA(epoch): 83s + Train loss: 0.853966 (914.7s) ETA: 6253min + Val loss: 0.852010 [t_0.0-0.2=1.0931 t_0.2-0.4=0.9601 t_0.4-0.6=0.8210 t_0.6-0.8=0.7097 t_0.8-1.0=0.6541] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0123 + Deleted old checkpoint: checkpoint_epoch_0120 +[MEM @ epoch 123 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 124/499 (25% done) --- + [Epoch 124][10/33] loss=0.850733 avg=0.861345 VRAM=39.9GiB | 24.9% done | ETA(epoch): 638s + [Epoch 124][20/33] loss=0.862184 avg=0.851249 VRAM=39.6GiB | 24.9% done | ETA(epoch): 360s + [Epoch 124][30/33] loss=0.823602 avg=0.847110 VRAM=39.9GiB | 25.0% done | ETA(epoch): 83s + Train loss: 0.845825 (914.2s) ETA: 6235min + Val loss: 0.868235 [t_0.0-0.2=1.0823 t_0.2-0.4=1.0001 t_0.4-0.6=0.8630 t_0.6-0.8=0.7206 t_0.8-1.0=0.6458] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0124 + Deleted old checkpoint: checkpoint_epoch_0121 +[MEM @ epoch 124 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 125/499 (25% done) --- + [Epoch 125][10/33] loss=0.843448 avg=0.858249 VRAM=39.9GiB | 25.1% done | ETA(epoch): 637s + [Epoch 125][20/33] loss=0.929005 avg=0.867824 VRAM=39.6GiB | 25.1% done | ETA(epoch): 360s + [Epoch 125][30/33] loss=0.880642 avg=0.867448 VRAM=39.9GiB | 25.2% done | ETA(epoch): 83s + Train loss: 0.865275 (914.2s) ETA: 6218min + Val loss: 0.869160 [t_0.0-0.2=1.0976 t_0.2-0.4=1.0082 t_0.4-0.6=0.8388 t_0.6-0.8=0.7324 t_0.8-1.0=0.6339] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0125 + Deleted old checkpoint: checkpoint_epoch_0122 +[MEM @ epoch 125 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 126/499 (25% done) --- + [Epoch 126][10/33] loss=0.968248 avg=0.889616 VRAM=39.9GiB | 25.3% done | ETA(epoch): 637s + [Epoch 126][20/33] loss=0.853012 avg=0.885406 VRAM=39.6GiB | 25.3% done | ETA(epoch): 360s + [Epoch 126][30/33] loss=0.841487 avg=0.872129 VRAM=39.9GiB | 25.4% done | ETA(epoch): 83s + Train loss: 0.866212 (914.4s) ETA: 6200min + Val loss: 0.823296 [t_0.0-0.2=1.0560 t_0.2-0.4=1.0207 t_0.4-0.6=0.8340 t_0.6-0.8=0.7345 t_0.8-1.0=0.6462] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0126 + Deleted old checkpoint: checkpoint_epoch_0123 +[MEM @ epoch 126 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 127/499 (25% done) --- + [Epoch 127][10/33] loss=0.804117 avg=0.846492 VRAM=39.9GiB | 25.5% done | ETA(epoch): 637s + [Epoch 127][20/33] loss=0.893767 avg=0.853007 VRAM=39.6GiB | 25.5% done | ETA(epoch): 360s + [Epoch 127][30/33] loss=0.905268 avg=0.851882 VRAM=39.9GiB | 25.6% done | ETA(epoch): 83s + Train loss: 0.852930 (914.2s) ETA: 6183min + Val loss: 0.868962 [t_0.0-0.2=1.0610 t_0.2-0.4=0.9947 t_0.4-0.6=0.8491 t_0.6-0.8=0.7342 t_0.8-1.0=0.6279] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0127 + Deleted old checkpoint: checkpoint_epoch_0124 +[MEM @ epoch 127 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 128/499 (26% done) --- + [Epoch 128][10/33] loss=0.814009 avg=0.843653 VRAM=39.9GiB | 25.7% done | ETA(epoch): 636s + [Epoch 128][20/33] loss=0.921230 avg=0.856581 VRAM=39.6GiB | 25.7% done | ETA(epoch): 360s + [Epoch 128][30/33] loss=0.930397 avg=0.854954 VRAM=39.9GiB | 25.8% done | ETA(epoch): 83s + Train loss: 0.853701 (913.9s) ETA: 6165min + Val loss: 0.879306 [t_0.0-0.2=1.0694 t_0.2-0.4=0.9650 t_0.4-0.6=0.8611 t_0.6-0.8=0.7161 t_0.8-1.0=0.6386] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0128 + Deleted old checkpoint: checkpoint_epoch_0125 +[MEM @ epoch 128 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 129/499 (26% done) --- + [Epoch 129][10/33] loss=0.896127 avg=0.880464 VRAM=39.9GiB | 25.9% done | ETA(epoch): 638s + [Epoch 129][20/33] loss=0.751274 avg=0.854779 VRAM=39.6GiB | 25.9% done | ETA(epoch): 360s + [Epoch 129][30/33] loss=0.892767 avg=0.852753 VRAM=39.9GiB | 26.0% done | ETA(epoch): 83s + Train loss: 0.851493 (915.0s) ETA: 6148min + Val loss: 0.832879 [t_0.0-0.2=1.0867 t_0.2-0.4=0.9905 t_0.4-0.6=0.8549 t_0.6-0.8=0.7149 t_0.8-1.0=0.6362] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0129 + Deleted old checkpoint: checkpoint_epoch_0126 +[MEM @ epoch 129 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 130/499 (26% done) --- + [Epoch 130][10/33] loss=0.874376 avg=0.850296 VRAM=39.9GiB | 26.1% done | ETA(epoch): 637s + [Epoch 130][20/33] loss=0.809088 avg=0.846493 VRAM=39.6GiB | 26.1% done | ETA(epoch): 360s + [Epoch 130][30/33] loss=0.891190 avg=0.845764 VRAM=39.9GiB | 26.2% done | ETA(epoch): 83s + Train loss: 0.842637 (914.4s) ETA: 6130min + Val loss: 0.885089 [t_0.0-0.2=1.0694 t_0.2-0.4=1.0161 t_0.4-0.6=0.8008 t_0.6-0.8=0.7111 t_0.8-1.0=0.6442] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0130 + Deleted old checkpoint: checkpoint_epoch_0127 +[MEM @ epoch 130 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 131/499 (26% done) --- + [Epoch 131][10/33] loss=0.823186 avg=0.848384 VRAM=39.9GiB | 26.3% done | ETA(epoch): 638s + [Epoch 131][20/33] loss=0.784696 avg=0.840482 VRAM=39.6GiB | 26.3% done | ETA(epoch): 360s + [Epoch 131][30/33] loss=0.798703 avg=0.847924 VRAM=39.9GiB | 26.4% done | ETA(epoch): 83s + Train loss: 0.847082 (915.3s) ETA: 6113min + Val loss: 0.841707 [t_0.0-0.2=1.0576 t_0.2-0.4=0.9958 t_0.4-0.6=0.8783 t_0.6-0.8=0.7242 t_0.8-1.0=0.6403] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0131 + Deleted old checkpoint: checkpoint_epoch_0128 +[MEM @ epoch 131 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 132/499 (26% done) --- + [Epoch 132][10/33] loss=0.957210 avg=0.870398 VRAM=39.9GiB | 26.5% done | ETA(epoch): 638s + [Epoch 132][20/33] loss=0.842618 avg=0.852035 VRAM=39.6GiB | 26.5% done | ETA(epoch): 360s + [Epoch 132][30/33] loss=0.800620 avg=0.852399 VRAM=39.9GiB | 26.6% done | ETA(epoch): 83s + Train loss: 0.850817 (914.7s) ETA: 6096min + Val loss: 0.846994 [t_0.0-0.2=1.0762 t_0.2-0.4=0.9744 t_0.4-0.6=0.8190 t_0.6-0.8=0.7390 t_0.8-1.0=0.6399] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0132 + Deleted old checkpoint: checkpoint_epoch_0129 +[MEM @ epoch 132 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 133/499 (27% done) --- + [Epoch 133][10/33] loss=0.823638 avg=0.868096 VRAM=39.9GiB | 26.7% done | ETA(epoch): 637s + [Epoch 133][20/33] loss=0.731820 avg=0.842280 VRAM=39.6GiB | 26.7% done | ETA(epoch): 360s + [Epoch 133][30/33] loss=0.785294 avg=0.845931 VRAM=39.9GiB | 26.8% done | ETA(epoch): 83s + Train loss: 0.848231 (914.5s) ETA: 6078min + Val loss: 0.870942 [t_0.0-0.2=1.0666 t_0.2-0.4=0.9767 t_0.4-0.6=0.8713 t_0.6-0.8=0.7288 t_0.8-1.0=0.6494] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0133 + Deleted old checkpoint: checkpoint_epoch_0130 +[MEM @ epoch 133 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 134/499 (27% done) --- + [Epoch 134][10/33] loss=0.761006 avg=0.872861 VRAM=39.9GiB | 26.9% done | ETA(epoch): 637s + [Epoch 134][20/33] loss=0.755783 avg=0.853341 VRAM=39.6GiB | 26.9% done | ETA(epoch): 360s + [Epoch 134][30/33] loss=0.954134 avg=0.859489 VRAM=39.9GiB | 27.0% done | ETA(epoch): 83s + Train loss: 0.858155 (914.8s) ETA: 6061min + Val loss: 0.854488 [t_0.0-0.2=1.0773 t_0.2-0.4=1.0200 t_0.4-0.6=0.8111 t_0.6-0.8=0.7094 t_0.8-1.0=0.6447] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0134 + Deleted old checkpoint: checkpoint_epoch_0131 +[MEM @ epoch 134 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 135/499 (27% done) --- + [Epoch 135][10/33] loss=0.892340 avg=0.864574 VRAM=39.9GiB | 27.1% done | ETA(epoch): 637s + [Epoch 135][20/33] loss=0.895259 avg=0.871753 VRAM=39.6GiB | 27.1% done | ETA(epoch): 360s + [Epoch 135][30/33] loss=0.711149 avg=0.855224 VRAM=39.9GiB | 27.2% done | ETA(epoch): 83s + Train loss: 0.861037 (915.2s) ETA: 6044min + Val loss: 0.859806 [t_0.0-0.2=1.0643 t_0.2-0.4=1.0047 t_0.4-0.6=0.8262 t_0.6-0.8=0.7137 t_0.8-1.0=0.6260] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0135 + Deleted old checkpoint: checkpoint_epoch_0132 +[MEM @ epoch 135 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 136/499 (27% done) --- + [Epoch 136][10/33] loss=0.912026 avg=0.849801 VRAM=39.9GiB | 27.3% done | ETA(epoch): 637s + [Epoch 136][20/33] loss=0.957085 avg=0.871579 VRAM=39.6GiB | 27.3% done | ETA(epoch): 361s + [Epoch 136][30/33] loss=0.814740 avg=0.861317 VRAM=39.9GiB | 27.4% done | ETA(epoch): 83s + Train loss: 0.861781 (915.3s) ETA: 6027min + Val loss: 0.856346 [t_0.0-0.2=1.0601 t_0.2-0.4=1.0141 t_0.4-0.6=0.8408 t_0.6-0.8=0.6991 t_0.8-1.0=0.6301] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0136 + Deleted old checkpoint: checkpoint_epoch_0133 +[MEM @ epoch 136 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 137/499 (27% done) --- + [Epoch 137][10/33] loss=0.849690 avg=0.840141 VRAM=39.9GiB | 27.5% done | ETA(epoch): 637s + [Epoch 137][20/33] loss=0.777467 avg=0.850722 VRAM=39.6GiB | 27.5% done | ETA(epoch): 360s + [Epoch 137][30/33] loss=0.829014 avg=0.851892 VRAM=39.9GiB | 27.6% done | ETA(epoch): 83s + Train loss: 0.848091 (915.0s) ETA: 6009min + Val loss: 0.849736 [t_0.0-0.2=1.0692 t_0.2-0.4=0.9339 t_0.4-0.6=0.8409 t_0.6-0.8=0.7449 t_0.8-1.0=0.6627] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0137 + Deleted old checkpoint: checkpoint_epoch_0134 +[MEM @ epoch 137 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 138/499 (28% done) --- + [Epoch 138][10/33] loss=0.839434 avg=0.838424 VRAM=39.9GiB | 27.7% done | ETA(epoch): 638s + [Epoch 138][20/33] loss=0.855243 avg=0.843129 VRAM=39.6GiB | 27.7% done | ETA(epoch): 361s + [Epoch 138][30/33] loss=0.840671 avg=0.841854 VRAM=39.9GiB | 27.8% done | ETA(epoch): 83s + Train loss: 0.845626 (915.7s) ETA: 5992min + Val loss: 0.880414 [t_0.0-0.2=1.0717 t_0.2-0.4=0.9991 t_0.4-0.6=0.8654 t_0.6-0.8=0.7077 t_0.8-1.0=0.6460] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0138 + Deleted old checkpoint: checkpoint_epoch_0135 +[MEM @ epoch 138 end] RAM: 18.1/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 139/499 (28% done) --- + [Epoch 139][10/33] loss=0.930155 avg=0.884543 VRAM=39.9GiB | 27.9% done | ETA(epoch): 638s + [Epoch 139][20/33] loss=0.864880 avg=0.861241 VRAM=39.6GiB | 27.9% done | ETA(epoch): 361s + [Epoch 139][30/33] loss=0.908811 avg=0.849672 VRAM=39.9GiB | 28.0% done | ETA(epoch): 83s + Train loss: 0.850393 (915.7s) ETA: 5975min + Val loss: 0.867354 [t_0.0-0.2=1.0692 t_0.2-0.4=1.0233 t_0.4-0.6=0.8478 t_0.6-0.8=0.7195 t_0.8-1.0=0.6536] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0139 + Deleted old checkpoint: checkpoint_epoch_0136 +[MEM @ epoch 139 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 140/499 (28% done) --- + [Epoch 140][10/33] loss=0.858721 avg=0.818865 VRAM=39.9GiB | 28.1% done | ETA(epoch): 638s + [Epoch 140][20/33] loss=0.892095 avg=0.835658 VRAM=39.6GiB | 28.1% done | ETA(epoch): 361s + [Epoch 140][30/33] loss=0.899839 avg=0.846807 VRAM=39.9GiB | 28.2% done | ETA(epoch): 83s + Train loss: 0.841633 (915.2s) ETA: 5958min + Val loss: 0.853254 [t_0.0-0.2=1.0443 t_0.2-0.4=0.9969 t_0.4-0.6=0.8653 t_0.6-0.8=0.7074 t_0.8-1.0=0.6288] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0140 + Deleted old checkpoint: checkpoint_epoch_0137 +[MEM @ epoch 140 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 141/499 (28% done) --- + [Epoch 141][10/33] loss=0.964790 avg=0.842051 VRAM=39.9GiB | 28.3% done | ETA(epoch): 637s + [Epoch 141][20/33] loss=0.783938 avg=0.839597 VRAM=39.6GiB | 28.3% done | ETA(epoch): 360s + [Epoch 141][30/33] loss=0.820987 avg=0.835993 VRAM=39.9GiB | 28.4% done | ETA(epoch): 83s + Train loss: 0.840485 (914.5s) ETA: 5940min + Val loss: 0.848954 [t_0.0-0.2=1.0664 t_0.2-0.4=1.0095 t_0.4-0.6=0.8359 t_0.6-0.8=0.7304 t_0.8-1.0=0.6444] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0141 + Deleted old checkpoint: checkpoint_epoch_0138 +[MEM @ epoch 141 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 142/499 (28% done) --- + [Epoch 142][10/33] loss=0.814365 avg=0.855541 VRAM=39.9GiB | 28.5% done | ETA(epoch): 637s + [Epoch 142][20/33] loss=0.888616 avg=0.865310 VRAM=39.6GiB | 28.5% done | ETA(epoch): 360s + [Epoch 142][30/33] loss=0.918142 avg=0.863941 VRAM=39.9GiB | 28.6% done | ETA(epoch): 83s + Train loss: 0.865396 (914.5s) ETA: 5923min + Val loss: 0.875411 [t_0.0-0.2=1.0833 t_0.2-0.4=0.9680 t_0.4-0.6=0.8511 t_0.6-0.8=0.7306 t_0.8-1.0=0.6451] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0142 + Deleted old checkpoint: checkpoint_epoch_0139 +[MEM @ epoch 142 end] RAM: 18.4/188.4 GiB (9.8%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 143/499 (29% done) --- + [Epoch 143][10/33] loss=0.919365 avg=0.882388 VRAM=39.9GiB | 28.7% done | ETA(epoch): 637s + [Epoch 143][20/33] loss=0.838654 avg=0.870393 VRAM=39.6GiB | 28.7% done | ETA(epoch): 360s + [Epoch 143][30/33] loss=0.812326 avg=0.857303 VRAM=39.9GiB | 28.8% done | ETA(epoch): 83s + Train loss: 0.857574 (914.8s) ETA: 5906min + Val loss: 0.832361 [t_0.0-0.2=1.0771 t_0.2-0.4=0.9979 t_0.4-0.6=0.7960 t_0.6-0.8=0.7149 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0143 + Deleted old checkpoint: checkpoint_epoch_0140 +[MEM @ epoch 143 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 144/499 (29% done) --- + [Epoch 144][10/33] loss=0.777283 avg=0.838550 VRAM=39.9GiB | 28.9% done | ETA(epoch): 637s + [Epoch 144][20/33] loss=0.829479 avg=0.842938 VRAM=39.6GiB | 28.9% done | ETA(epoch): 360s + [Epoch 144][30/33] loss=0.854676 avg=0.843483 VRAM=39.9GiB | 29.0% done | ETA(epoch): 83s + Train loss: 0.844577 (914.1s) ETA: 5889min + Val loss: 0.841149 [t_0.0-0.2=1.0809 t_0.2-0.4=0.9945 t_0.4-0.6=0.8584 t_0.6-0.8=0.7094 t_0.8-1.0=0.6360] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0144 + Deleted old checkpoint: checkpoint_epoch_0141 +[MEM @ epoch 144 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 145/499 (29% done) --- + [Epoch 145][10/33] loss=0.845168 avg=0.825704 VRAM=39.9GiB | 29.1% done | ETA(epoch): 637s + [Epoch 145][20/33] loss=0.881865 avg=0.855653 VRAM=39.6GiB | 29.1% done | ETA(epoch): 360s + [Epoch 145][30/33] loss=0.761571 avg=0.857883 VRAM=39.9GiB | 29.2% done | ETA(epoch): 83s + Train loss: 0.855630 (913.7s) ETA: 5872min + Val loss: 0.839148 [t_0.0-0.2=1.0712 t_0.2-0.4=1.0195 t_0.4-0.6=0.8179 t_0.6-0.8=0.7271 t_0.8-1.0=0.6342] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0145 + Deleted old checkpoint: checkpoint_epoch_0142 +[MEM @ epoch 145 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 146/499 (29% done) --- + [Epoch 146][10/33] loss=0.908260 avg=0.849058 VRAM=39.9GiB | 29.3% done | ETA(epoch): 636s + [Epoch 146][20/33] loss=0.884717 avg=0.850566 VRAM=39.6GiB | 29.3% done | ETA(epoch): 360s + [Epoch 146][30/33] loss=0.906183 avg=0.862452 VRAM=39.9GiB | 29.4% done | ETA(epoch): 83s + Train loss: 0.870221 (913.8s) ETA: 5854min + Val loss: 0.885061 [t_0.0-0.2=1.0705 t_0.2-0.4=0.9905 t_0.4-0.6=0.8444 t_0.6-0.8=0.7119 t_0.8-1.0=0.6284] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0146 + Deleted old checkpoint: checkpoint_epoch_0143 +[MEM @ epoch 146 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 147/499 (29% done) --- + [Epoch 147][10/33] loss=0.866268 avg=0.885289 VRAM=39.9GiB | 29.5% done | ETA(epoch): 637s + [Epoch 147][20/33] loss=0.836164 avg=0.861878 VRAM=39.6GiB | 29.5% done | ETA(epoch): 360s + [Epoch 147][30/33] loss=0.822853 avg=0.852196 VRAM=39.9GiB | 29.6% done | ETA(epoch): 83s + Train loss: 0.852062 (913.7s) ETA: 5837min + Val loss: 0.859803 [t_0.0-0.2=1.0630 t_0.2-0.4=0.9906 t_0.4-0.6=0.8746 t_0.6-0.8=0.7130 t_0.8-1.0=0.6620] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0147 + Deleted old checkpoint: checkpoint_epoch_0144 +[MEM @ epoch 147 end] RAM: 18.3/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 148/499 (30% done) --- + [Epoch 148][10/33] loss=0.900376 avg=0.838388 VRAM=39.9GiB | 29.7% done | ETA(epoch): 637s + [Epoch 148][20/33] loss=0.833490 avg=0.835022 VRAM=39.6GiB | 29.7% done | ETA(epoch): 360s + [Epoch 148][30/33] loss=0.829032 avg=0.835035 VRAM=39.9GiB | 29.8% done | ETA(epoch): 83s + Train loss: 0.834398 (914.0s) ETA: 5820min + Val loss: 0.901166 [t_0.0-0.2=1.0693 t_0.2-0.4=1.0341 t_0.4-0.6=0.8483 t_0.6-0.8=0.7090 t_0.8-1.0=0.6494] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0148 + Deleted old checkpoint: checkpoint_epoch_0145 +[MEM @ epoch 148 end] RAM: 18.2/188.4 GiB (9.7%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 149/499 (30% done) --- + [Epoch 149][10/33] loss=0.794351 avg=0.867374 VRAM=39.9GiB | 29.9% done | ETA(epoch): 636s + [Epoch 149][20/33] loss=0.851947 avg=0.865690 VRAM=39.6GiB | 29.9% done | ETA(epoch): 360s + [Epoch 149][30/33] loss=0.837008 avg=0.869087 VRAM=39.9GiB | 30.0% done | ETA(epoch): 83s + Train loss: 0.867973 (913.5s) ETA: 5803min + Val loss: 0.864510 [t_0.0-0.2=1.0791 t_0.2-0.4=0.9779 t_0.4-0.6=0.8419 t_0.6-0.8=0.7132 t_0.8-1.0=0.6475] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0149 + Deleted old checkpoint: checkpoint_epoch_0146 +[MEM @ epoch 149 end] RAM: 18.2/188.4 GiB (9.6%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 150/499 (30% done) --- + [MilestoneVis] train_0 step 4950 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 4950 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 4950 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 4950 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 4950 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 4950 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 4950 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 4950 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 4950 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 4950 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 150][10/33] loss=0.908427 avg=0.851002 VRAM=39.9GiB | 30.1% done | ETA(epoch): 2369s + [Epoch 150][20/33] loss=0.792734 avg=0.843198 VRAM=39.6GiB | 30.1% done | ETA(epoch): 849s + [Epoch 150][30/33] loss=0.916402 avg=0.847027 VRAM=39.9GiB | 30.2% done | ETA(epoch): 158s + Train loss: 0.847243 (1666.8s) ETA: 5815min + Val loss: 0.840133 [t_0.0-0.2=1.0760 t_0.2-0.4=0.9884 t_0.4-0.6=0.8315 t_0.6-0.8=0.7065 t_0.8-1.0=0.6358] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0150 + Deleted old checkpoint: checkpoint_epoch_0147 +[MEM @ epoch 150 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 151/499 (30% done) --- + [Epoch 151][10/33] loss=0.866313 avg=0.887222 VRAM=39.9GiB | 30.3% done | ETA(epoch): 637s + [Epoch 151][20/33] loss=0.848494 avg=0.876275 VRAM=39.6GiB | 30.3% done | ETA(epoch): 360s + [Epoch 151][30/33] loss=0.865090 avg=0.875518 VRAM=39.9GiB | 30.4% done | ETA(epoch): 83s + Train loss: 0.870911 (913.7s) ETA: 5797min + Val loss: 0.867339 [t_0.0-0.2=1.0663 t_0.2-0.4=0.9809 t_0.4-0.6=0.8784 t_0.6-0.8=0.6949 t_0.8-1.0=0.6548] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0151 + Deleted old checkpoint: checkpoint_epoch_0148 +[MEM @ epoch 151 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 152/499 (30% done) --- + [Epoch 152][10/33] loss=0.952044 avg=0.864908 VRAM=39.9GiB | 30.5% done | ETA(epoch): 637s + [Epoch 152][20/33] loss=0.933748 avg=0.857135 VRAM=39.6GiB | 30.5% done | ETA(epoch): 360s + [Epoch 152][30/33] loss=0.864968 avg=0.850301 VRAM=39.9GiB | 30.6% done | ETA(epoch): 83s + Train loss: 0.853850 (913.4s) ETA: 5780min + Val loss: 0.792904 [t_0.0-0.2=1.0813 t_0.2-0.4=0.9621 t_0.4-0.6=0.8450 t_0.6-0.8=0.6998 t_0.8-1.0=0.6344] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0152 (BEST) + Deleted old checkpoint: checkpoint_epoch_0110 + Deleted old checkpoint: checkpoint_epoch_0149 +[MEM @ epoch 152 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 153/499 (31% done) --- + [Epoch 153][10/33] loss=0.821008 avg=0.852602 VRAM=39.9GiB | 30.7% done | ETA(epoch): 636s + [Epoch 153][20/33] loss=0.862713 avg=0.847937 VRAM=39.6GiB | 30.7% done | ETA(epoch): 360s + [Epoch 153][30/33] loss=0.927811 avg=0.855650 VRAM=39.9GiB | 30.8% done | ETA(epoch): 83s + Train loss: 0.850917 (913.6s) ETA: 5762min + Val loss: 0.832046 [t_0.0-0.2=1.0495 t_0.2-0.4=0.9992 t_0.4-0.6=0.8488 t_0.6-0.8=0.7195 t_0.8-1.0=0.6464] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0153 + Deleted old checkpoint: checkpoint_epoch_0150 +[MEM @ epoch 153 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 154/499 (31% done) --- + [Epoch 154][10/33] loss=0.879498 avg=0.866435 VRAM=39.9GiB | 30.9% done | ETA(epoch): 636s + [Epoch 154][20/33] loss=0.880397 avg=0.854551 VRAM=39.6GiB | 30.9% done | ETA(epoch): 360s + [Epoch 154][30/33] loss=0.913173 avg=0.856693 VRAM=39.9GiB | 31.0% done | ETA(epoch): 83s + Train loss: 0.861402 (912.9s) ETA: 5745min + Val loss: 0.823333 [t_0.0-0.2=1.0669 t_0.2-0.4=1.0373 t_0.4-0.6=0.8356 t_0.6-0.8=0.6947 t_0.8-1.0=0.6384] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0154 + Deleted old checkpoint: checkpoint_epoch_0151 +[MEM @ epoch 154 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 155/499 (31% done) --- + [Epoch 155][10/33] loss=0.862484 avg=0.888897 VRAM=39.9GiB | 31.1% done | ETA(epoch): 637s + [Epoch 155][20/33] loss=0.801038 avg=0.869019 VRAM=39.6GiB | 31.1% done | ETA(epoch): 360s + [Epoch 155][30/33] loss=0.976304 avg=0.872355 VRAM=39.9GiB | 31.2% done | ETA(epoch): 83s + Train loss: 0.869941 (913.5s) ETA: 5728min + Val loss: 0.862625 [t_0.0-0.2=1.0550 t_0.2-0.4=1.0132 t_0.4-0.6=0.8460 t_0.6-0.8=0.7225 t_0.8-1.0=0.6331] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0155 +[MEM @ epoch 155 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 156/499 (31% done) --- + [Epoch 156][10/33] loss=0.845340 avg=0.859136 VRAM=39.9GiB | 31.3% done | ETA(epoch): 635s + [Epoch 156][20/33] loss=0.924538 avg=0.861722 VRAM=39.6GiB | 31.3% done | ETA(epoch): 359s + [Epoch 156][30/33] loss=0.766875 avg=0.852420 VRAM=39.9GiB | 31.4% done | ETA(epoch): 83s + Train loss: 0.846126 (912.5s) ETA: 5710min + Val loss: 0.842274 [t_0.0-0.2=1.0762 t_0.2-0.4=1.0166 t_0.4-0.6=0.8222 t_0.6-0.8=0.7080 t_0.8-1.0=0.6512] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0156 + Deleted old checkpoint: checkpoint_epoch_0153 +[MEM @ epoch 156 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 157/499 (31% done) --- + [Epoch 157][10/33] loss=0.901965 avg=0.839509 VRAM=39.9GiB | 31.5% done | ETA(epoch): 636s + [Epoch 157][20/33] loss=0.786656 avg=0.848648 VRAM=39.6GiB | 31.5% done | ETA(epoch): 359s + [Epoch 157][30/33] loss=0.815320 avg=0.853659 VRAM=39.9GiB | 31.6% done | ETA(epoch): 83s + Train loss: 0.858225 (912.6s) ETA: 5693min + Val loss: 0.852433 [t_0.0-0.2=1.0854 t_0.2-0.4=0.9927 t_0.4-0.6=0.8231 t_0.6-0.8=0.7164 t_0.8-1.0=0.6363] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0157 + Deleted old checkpoint: checkpoint_epoch_0154 +[MEM @ epoch 157 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 158/499 (32% done) --- + [Epoch 158][10/33] loss=0.948544 avg=0.876731 VRAM=39.9GiB | 31.7% done | ETA(epoch): 636s + [Epoch 158][20/33] loss=0.863127 avg=0.877160 VRAM=39.6GiB | 31.7% done | ETA(epoch): 359s + [Epoch 158][30/33] loss=0.773946 avg=0.863612 VRAM=39.9GiB | 31.8% done | ETA(epoch): 83s + Train loss: 0.861163 (912.3s) ETA: 5675min + Val loss: 0.847340 [t_0.0-0.2=1.0641 t_0.2-0.4=0.9659 t_0.4-0.6=0.8198 t_0.6-0.8=0.7251 t_0.8-1.0=0.6248] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0158 + Deleted old checkpoint: checkpoint_epoch_0155 +[MEM @ epoch 158 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 159/499 (32% done) --- + [Epoch 159][10/33] loss=0.808595 avg=0.854327 VRAM=39.9GiB | 31.9% done | ETA(epoch): 636s + [Epoch 159][20/33] loss=0.771422 avg=0.845750 VRAM=39.6GiB | 31.9% done | ETA(epoch): 359s + [Epoch 159][30/33] loss=0.977364 avg=0.856594 VRAM=39.9GiB | 32.0% done | ETA(epoch): 83s + Train loss: 0.854343 (912.3s) ETA: 5658min + Val loss: 0.853360 [t_0.0-0.2=1.0767 t_0.2-0.4=0.9839 t_0.4-0.6=0.8162 t_0.6-0.8=0.7063 t_0.8-1.0=0.6312] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0159 + Deleted old checkpoint: checkpoint_epoch_0156 +[MEM @ epoch 159 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 160/499 (32% done) --- + [Epoch 160][10/33] loss=0.819648 avg=0.850359 VRAM=39.9GiB | 32.1% done | ETA(epoch): 636s + [Epoch 160][20/33] loss=0.952115 avg=0.864630 VRAM=39.6GiB | 32.1% done | ETA(epoch): 359s + [Epoch 160][30/33] loss=0.837753 avg=0.862215 VRAM=39.9GiB | 32.2% done | ETA(epoch): 83s + Train loss: 0.858275 (912.3s) ETA: 5641min + Val loss: 0.857727 [t_0.0-0.2=1.0608 t_0.2-0.4=0.9904 t_0.4-0.6=0.8594 t_0.6-0.8=0.7047 t_0.8-1.0=0.6349] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0160 + Deleted old checkpoint: checkpoint_epoch_0157 +[MEM @ epoch 160 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 161/499 (32% done) --- + [Epoch 161][10/33] loss=0.861725 avg=0.819224 VRAM=39.9GiB | 32.3% done | ETA(epoch): 635s + [Epoch 161][20/33] loss=0.892694 avg=0.833250 VRAM=39.6GiB | 32.3% done | ETA(epoch): 359s + [Epoch 161][30/33] loss=0.961255 avg=0.850304 VRAM=39.9GiB | 32.4% done | ETA(epoch): 83s + Train loss: 0.847618 (912.2s) ETA: 5623min + Val loss: 0.838047 [t_0.0-0.2=1.0745 t_0.2-0.4=0.9953 t_0.4-0.6=0.8564 t_0.6-0.8=0.7188 t_0.8-1.0=0.6355] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0161 + Deleted old checkpoint: checkpoint_epoch_0158 +[MEM @ epoch 161 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 162/499 (32% done) --- + [Epoch 162][10/33] loss=0.831511 avg=0.862973 VRAM=39.9GiB | 32.5% done | ETA(epoch): 635s + [Epoch 162][20/33] loss=0.959611 avg=0.859801 VRAM=39.6GiB | 32.5% done | ETA(epoch): 359s + [Epoch 162][30/33] loss=0.773588 avg=0.853959 VRAM=39.9GiB | 32.6% done | ETA(epoch): 83s + Train loss: 0.847765 (912.1s) ETA: 5606min + Val loss: 0.831447 [t_0.0-0.2=1.0882 t_0.2-0.4=0.9927 t_0.4-0.6=0.8397 t_0.6-0.8=0.7319 t_0.8-1.0=0.6304] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0162 + Deleted old checkpoint: checkpoint_epoch_0159 +[MEM @ epoch 162 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 163/499 (33% done) --- + [Epoch 163][10/33] loss=0.842425 avg=0.867048 VRAM=39.9GiB | 32.7% done | ETA(epoch): 636s + [Epoch 163][20/33] loss=0.740364 avg=0.853932 VRAM=39.6GiB | 32.7% done | ETA(epoch): 360s + [Epoch 163][30/33] loss=0.798245 avg=0.852941 VRAM=39.9GiB | 32.8% done | ETA(epoch): 83s + Train loss: 0.856553 (912.8s) ETA: 5589min + Val loss: 0.880639 [t_0.0-0.2=1.0727 t_0.2-0.4=0.9878 t_0.4-0.6=0.8717 t_0.6-0.8=0.6899 t_0.8-1.0=0.6483] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0163 + Deleted old checkpoint: checkpoint_epoch_0160 +[MEM @ epoch 163 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 164/499 (33% done) --- + [Epoch 164][10/33] loss=0.929985 avg=0.857366 VRAM=39.9GiB | 32.9% done | ETA(epoch): 636s + [Epoch 164][20/33] loss=0.781855 avg=0.853712 VRAM=39.6GiB | 32.9% done | ETA(epoch): 359s + [Epoch 164][30/33] loss=0.847832 avg=0.850727 VRAM=39.9GiB | 33.0% done | ETA(epoch): 83s + Train loss: 0.844899 (912.4s) ETA: 5571min + Val loss: 0.871681 [t_0.0-0.2=1.0675 t_0.2-0.4=1.0063 t_0.4-0.6=0.8554 t_0.6-0.8=0.7071 t_0.8-1.0=0.6359] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0164 + Deleted old checkpoint: checkpoint_epoch_0161 +[MEM @ epoch 164 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 165/499 (33% done) --- + [Epoch 165][10/33] loss=0.888174 avg=0.836436 VRAM=39.9GiB | 33.1% done | ETA(epoch): 636s + [Epoch 165][20/33] loss=0.944748 avg=0.845863 VRAM=39.6GiB | 33.1% done | ETA(epoch): 360s + [Epoch 165][30/33] loss=0.796935 avg=0.842894 VRAM=39.9GiB | 33.2% done | ETA(epoch): 83s + Train loss: 0.849606 (913.1s) ETA: 5554min + Val loss: 0.838158 [t_0.0-0.2=1.0780 t_0.2-0.4=0.9698 t_0.4-0.6=0.8428 t_0.6-0.8=0.7060 t_0.8-1.0=0.6081] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0165 + Deleted old checkpoint: checkpoint_epoch_0162 +[MEM @ epoch 165 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 166/499 (33% done) --- + [Epoch 166][10/33] loss=0.932362 avg=0.853412 VRAM=39.9GiB | 33.3% done | ETA(epoch): 636s + [Epoch 166][20/33] loss=0.846193 avg=0.833081 VRAM=39.6GiB | 33.3% done | ETA(epoch): 360s + [Epoch 166][30/33] loss=0.977996 avg=0.845807 VRAM=39.9GiB | 33.4% done | ETA(epoch): 83s + Train loss: 0.846166 (913.1s) ETA: 5537min + Val loss: 0.865569 [t_0.0-0.2=1.0808 t_0.2-0.4=0.9924 t_0.4-0.6=0.8300 t_0.6-0.8=0.6890 t_0.8-1.0=0.6425] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0166 + Deleted old checkpoint: checkpoint_epoch_0163 +[MEM @ epoch 166 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 167/499 (33% done) --- + [Epoch 167][10/33] loss=0.842671 avg=0.856010 VRAM=39.9GiB | 33.5% done | ETA(epoch): 636s + [Epoch 167][20/33] loss=0.927736 avg=0.864916 VRAM=39.6GiB | 33.5% done | ETA(epoch): 360s + [Epoch 167][30/33] loss=0.833365 avg=0.869880 VRAM=39.9GiB | 33.6% done | ETA(epoch): 83s + Train loss: 0.865336 (912.9s) ETA: 5520min + Val loss: 0.850715 [t_0.0-0.2=1.0746 t_0.2-0.4=0.9894 t_0.4-0.6=0.8631 t_0.6-0.8=0.6919 t_0.8-1.0=0.6307] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0167 + Deleted old checkpoint: checkpoint_epoch_0164 +[MEM @ epoch 167 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 168/499 (34% done) --- + [Epoch 168][10/33] loss=0.907564 avg=0.854135 VRAM=39.9GiB | 33.7% done | ETA(epoch): 636s + [Epoch 168][20/33] loss=0.843458 avg=0.843698 VRAM=39.6GiB | 33.7% done | ETA(epoch): 359s + [Epoch 168][30/33] loss=0.805356 avg=0.841602 VRAM=39.9GiB | 33.8% done | ETA(epoch): 83s + Train loss: 0.843183 (912.3s) ETA: 5502min + Val loss: 0.865782 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9572 t_0.4-0.6=0.8432 t_0.6-0.8=0.7314 t_0.8-1.0=0.6370] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0168 + Deleted old checkpoint: checkpoint_epoch_0165 +[MEM @ epoch 168 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 169/499 (34% done) --- + [Epoch 169][10/33] loss=0.825275 avg=0.836729 VRAM=39.9GiB | 33.9% done | ETA(epoch): 636s + [Epoch 169][20/33] loss=0.933479 avg=0.840058 VRAM=39.6GiB | 33.9% done | ETA(epoch): 360s + [Epoch 169][30/33] loss=0.810237 avg=0.851672 VRAM=39.9GiB | 34.0% done | ETA(epoch): 83s + Train loss: 0.848514 (913.2s) ETA: 5485min + Val loss: 0.883362 [t_0.0-0.2=1.0853 t_0.2-0.4=0.9747 t_0.4-0.6=0.8320 t_0.6-0.8=0.6805 t_0.8-1.0=0.6437] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0169 + Deleted old checkpoint: checkpoint_epoch_0166 +[MEM @ epoch 169 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 170/499 (34% done) --- + [Epoch 170][10/33] loss=0.806064 avg=0.859135 VRAM=39.9GiB | 34.1% done | ETA(epoch): 636s + [Epoch 170][20/33] loss=0.842411 avg=0.855432 VRAM=39.6GiB | 34.1% done | ETA(epoch): 360s + [Epoch 170][30/33] loss=0.901048 avg=0.857574 VRAM=39.9GiB | 34.2% done | ETA(epoch): 83s + Train loss: 0.861828 (913.0s) ETA: 5468min + Val loss: 0.844062 [t_0.0-0.2=1.0730 t_0.2-0.4=1.0141 t_0.4-0.6=0.8273 t_0.6-0.8=0.7060 t_0.8-1.0=0.6176] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0170 + Deleted old checkpoint: checkpoint_epoch_0167 +[MEM @ epoch 170 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 171/499 (34% done) --- + [Epoch 171][10/33] loss=0.907753 avg=0.872382 VRAM=39.9GiB | 34.3% done | ETA(epoch): 636s + [Epoch 171][20/33] loss=0.870874 avg=0.865260 VRAM=39.6GiB | 34.3% done | ETA(epoch): 359s + [Epoch 171][30/33] loss=0.938808 avg=0.864377 VRAM=39.9GiB | 34.4% done | ETA(epoch): 83s + Train loss: 0.859961 (912.9s) ETA: 5451min + Val loss: 0.883005 [t_0.0-0.2=1.0789 t_0.2-0.4=0.9680 t_0.4-0.6=0.8519 t_0.6-0.8=0.7134 t_0.8-1.0=0.6520] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0171 + Deleted old checkpoint: checkpoint_epoch_0168 +[MEM @ epoch 171 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 172/499 (34% done) --- + [Epoch 172][10/33] loss=0.885513 avg=0.871127 VRAM=39.9GiB | 34.5% done | ETA(epoch): 636s + [Epoch 172][20/33] loss=0.856279 avg=0.846774 VRAM=39.6GiB | 34.5% done | ETA(epoch): 360s + [Epoch 172][30/33] loss=0.856768 avg=0.846348 VRAM=39.9GiB | 34.6% done | ETA(epoch): 83s + Train loss: 0.848917 (912.9s) ETA: 5434min + Val loss: 0.876442 [t_0.0-0.2=1.0777 t_0.2-0.4=1.0305 t_0.4-0.6=0.8402 t_0.6-0.8=0.7145 t_0.8-1.0=0.6402] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0172 + Deleted old checkpoint: checkpoint_epoch_0169 +[MEM @ epoch 172 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 173/499 (35% done) --- + [Epoch 173][10/33] loss=0.887031 avg=0.868968 VRAM=39.9GiB | 34.7% done | ETA(epoch): 636s + [Epoch 173][20/33] loss=0.868921 avg=0.861408 VRAM=39.6GiB | 34.7% done | ETA(epoch): 360s + [Epoch 173][30/33] loss=0.862794 avg=0.855844 VRAM=39.9GiB | 34.8% done | ETA(epoch): 83s + Train loss: 0.856450 (912.8s) ETA: 5416min + Val loss: 0.857663 [t_0.0-0.2=1.0819 t_0.2-0.4=0.9854 t_0.4-0.6=0.8669 t_0.6-0.8=0.7207 t_0.8-1.0=0.6364] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0173 + Deleted old checkpoint: checkpoint_epoch_0170 +[MEM @ epoch 173 end] RAM: 18.7/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 174/499 (35% done) --- + [Epoch 174][10/33] loss=0.874437 avg=0.873086 VRAM=39.9GiB | 34.9% done | ETA(epoch): 636s + [Epoch 174][20/33] loss=0.802048 avg=0.867351 VRAM=39.6GiB | 34.9% done | ETA(epoch): 359s + [Epoch 174][30/33] loss=0.850642 avg=0.858635 VRAM=39.9GiB | 35.0% done | ETA(epoch): 83s + Train loss: 0.854634 (912.6s) ETA: 5399min + Val loss: 0.859950 [t_0.0-0.2=1.0828 t_0.2-0.4=1.0398 t_0.4-0.6=0.8408 t_0.6-0.8=0.7084 t_0.8-1.0=0.6256] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0174 + Deleted old checkpoint: checkpoint_epoch_0171 +[MEM @ epoch 174 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 175/499 (35% done) --- + [Epoch 175][10/33] loss=0.940096 avg=0.866765 VRAM=39.9GiB | 35.1% done | ETA(epoch): 636s + [Epoch 175][20/33] loss=0.933637 avg=0.868149 VRAM=39.6GiB | 35.1% done | ETA(epoch): 360s + [Epoch 175][30/33] loss=0.882451 avg=0.868313 VRAM=39.9GiB | 35.2% done | ETA(epoch): 83s + Train loss: 0.863948 (913.0s) ETA: 5382min + Val loss: 0.813892 [t_0.0-0.2=1.0736 t_0.2-0.4=0.9823 t_0.4-0.6=0.8579 t_0.6-0.8=0.7185 t_0.8-1.0=0.6391] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0175 + Deleted old checkpoint: checkpoint_epoch_0172 +[MEM @ epoch 175 end] RAM: 18.6/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 176/499 (35% done) --- + [Epoch 176][10/33] loss=0.848879 avg=0.837581 VRAM=39.9GiB | 35.3% done | ETA(epoch): 636s + [Epoch 176][20/33] loss=0.917259 avg=0.851025 VRAM=39.6GiB | 35.3% done | ETA(epoch): 359s + [Epoch 176][30/33] loss=0.890481 avg=0.844698 VRAM=39.9GiB | 35.4% done | ETA(epoch): 83s + Train loss: 0.845715 (912.6s) ETA: 5365min + Val loss: 0.847422 [t_0.0-0.2=1.0662 t_0.2-0.4=0.9916 t_0.4-0.6=0.8579 t_0.6-0.8=0.7128 t_0.8-1.0=0.6415] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0176 + Deleted old checkpoint: checkpoint_epoch_0173 +[MEM @ epoch 176 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 177/499 (35% done) --- + [Epoch 177][10/33] loss=0.923520 avg=0.855720 VRAM=39.9GiB | 35.5% done | ETA(epoch): 636s + [Epoch 177][20/33] loss=0.776767 avg=0.846595 VRAM=39.6GiB | 35.5% done | ETA(epoch): 359s + [Epoch 177][30/33] loss=0.953663 avg=0.851709 VRAM=39.9GiB | 35.6% done | ETA(epoch): 83s + Train loss: 0.854466 (912.0s) ETA: 5348min + Val loss: 0.837549 [t_0.0-0.2=1.0722 t_0.2-0.4=1.0058 t_0.4-0.6=0.8508 t_0.6-0.8=0.7143 t_0.8-1.0=0.6213] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0177 + Deleted old checkpoint: checkpoint_epoch_0174 +[MEM @ epoch 177 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 178/499 (36% done) --- + [Epoch 178][10/33] loss=0.809805 avg=0.844277 VRAM=39.9GiB | 35.7% done | ETA(epoch): 636s + [Epoch 178][20/33] loss=0.956609 avg=0.857115 VRAM=39.6GiB | 35.7% done | ETA(epoch): 360s + [Epoch 178][30/33] loss=0.882860 avg=0.850807 VRAM=39.9GiB | 35.8% done | ETA(epoch): 83s + Train loss: 0.851450 (913.2s) ETA: 5331min + Val loss: 0.861254 [t_0.0-0.2=1.0577 t_0.2-0.4=0.9632 t_0.4-0.6=0.8695 t_0.6-0.8=0.6726 t_0.8-1.0=0.6316] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0178 + Deleted old checkpoint: checkpoint_epoch_0175 +[MEM @ epoch 178 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 179/499 (36% done) --- + [Epoch 179][10/33] loss=0.820530 avg=0.848893 VRAM=39.9GiB | 35.9% done | ETA(epoch): 636s + [Epoch 179][20/33] loss=0.889285 avg=0.865195 VRAM=39.6GiB | 35.9% done | ETA(epoch): 360s + [Epoch 179][30/33] loss=0.832337 avg=0.863437 VRAM=39.9GiB | 36.0% done | ETA(epoch): 83s + Train loss: 0.864554 (913.3s) ETA: 5313min + Val loss: 0.843934 [t_0.0-0.2=1.0704 t_0.2-0.4=0.9845 t_0.4-0.6=0.8709 t_0.6-0.8=0.6752 t_0.8-1.0=0.6243] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0179 + Deleted old checkpoint: checkpoint_epoch_0176 +[MEM @ epoch 179 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 180/499 (36% done) --- + [Epoch 180][10/33] loss=0.834802 avg=0.850637 VRAM=39.9GiB | 36.1% done | ETA(epoch): 636s + [Epoch 180][20/33] loss=0.827118 avg=0.841575 VRAM=39.6GiB | 36.1% done | ETA(epoch): 359s + [Epoch 180][30/33] loss=0.940759 avg=0.847741 VRAM=39.9GiB | 36.2% done | ETA(epoch): 83s + Train loss: 0.844453 (912.9s) ETA: 5296min + Val loss: 0.840664 [t_0.0-0.2=1.0536 t_0.2-0.4=0.9914 t_0.4-0.6=0.8510 t_0.6-0.8=0.7173 t_0.8-1.0=0.6304] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0180 + Deleted old checkpoint: checkpoint_epoch_0177 +[MEM @ epoch 180 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 181/499 (36% done) --- + [Epoch 181][10/33] loss=0.935652 avg=0.880527 VRAM=39.9GiB | 36.3% done | ETA(epoch): 637s + [Epoch 181][20/33] loss=0.843327 avg=0.869139 VRAM=39.6GiB | 36.3% done | ETA(epoch): 360s + [Epoch 181][30/33] loss=0.921437 avg=0.865403 VRAM=39.9GiB | 36.4% done | ETA(epoch): 83s + Train loss: 0.865464 (913.5s) ETA: 5279min + Val loss: 0.861345 [t_0.0-0.2=1.0739 t_0.2-0.4=0.9764 t_0.4-0.6=0.8667 t_0.6-0.8=0.7213 t_0.8-1.0=0.6377] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0181 + Deleted old checkpoint: checkpoint_epoch_0178 +[MEM @ epoch 181 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 182/499 (36% done) --- + [Epoch 182][10/33] loss=0.820148 avg=0.843454 VRAM=39.9GiB | 36.5% done | ETA(epoch): 636s + [Epoch 182][20/33] loss=0.884118 avg=0.855076 VRAM=39.6GiB | 36.5% done | ETA(epoch): 360s + [Epoch 182][30/33] loss=0.826232 avg=0.850145 VRAM=39.9GiB | 36.6% done | ETA(epoch): 83s + Train loss: 0.849429 (913.0s) ETA: 5262min + Val loss: 0.846841 [t_0.0-0.2=1.0706 t_0.2-0.4=0.9933 t_0.4-0.6=0.8196 t_0.6-0.8=0.7073 t_0.8-1.0=0.6375] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0182 + Deleted old checkpoint: checkpoint_epoch_0179 +[MEM @ epoch 182 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 183/499 (37% done) --- + [Epoch 183][10/33] loss=0.765518 avg=0.833674 VRAM=39.9GiB | 36.7% done | ETA(epoch): 636s + [Epoch 183][20/33] loss=0.891302 avg=0.847255 VRAM=39.6GiB | 36.7% done | ETA(epoch): 360s + [Epoch 183][30/33] loss=0.854159 avg=0.835073 VRAM=39.9GiB | 36.8% done | ETA(epoch): 83s + Train loss: 0.838259 (913.1s) ETA: 5245min + Val loss: 0.826359 [t_0.0-0.2=1.0693 t_0.2-0.4=0.9822 t_0.4-0.6=0.8547 t_0.6-0.8=0.7197 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0183 + Deleted old checkpoint: checkpoint_epoch_0180 +[MEM @ epoch 183 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 184/499 (37% done) --- + [Epoch 184][10/33] loss=0.838096 avg=0.856706 VRAM=39.9GiB | 36.9% done | ETA(epoch): 637s + [Epoch 184][20/33] loss=0.784403 avg=0.859436 VRAM=39.6GiB | 36.9% done | ETA(epoch): 360s + [Epoch 184][30/33] loss=0.910125 avg=0.863198 VRAM=39.9GiB | 37.0% done | ETA(epoch): 83s + Train loss: 0.861620 (913.4s) ETA: 5228min + Val loss: 0.830802 [t_0.0-0.2=1.0667 t_0.2-0.4=0.9745 t_0.4-0.6=0.8058 t_0.6-0.8=0.7051 t_0.8-1.0=0.6401] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0184 + Deleted old checkpoint: checkpoint_epoch_0181 +[MEM @ epoch 184 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 185/499 (37% done) --- + [Epoch 185][10/33] loss=0.950901 avg=0.872568 VRAM=39.9GiB | 37.1% done | ETA(epoch): 637s + [Epoch 185][20/33] loss=0.834229 avg=0.852613 VRAM=39.6GiB | 37.1% done | ETA(epoch): 360s + [Epoch 185][30/33] loss=0.837935 avg=0.847234 VRAM=39.9GiB | 37.2% done | ETA(epoch): 83s + Train loss: 0.850667 (912.9s) ETA: 5211min + Val loss: 0.857072 [t_0.0-0.2=1.0823 t_0.2-0.4=0.9712 t_0.4-0.6=0.8245 t_0.6-0.8=0.7239 t_0.8-1.0=0.6473] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0185 + Deleted old checkpoint: checkpoint_epoch_0182 +[MEM @ epoch 185 end] RAM: 18.9/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 186/499 (37% done) --- + [Epoch 186][10/33] loss=0.843200 avg=0.851086 VRAM=39.9GiB | 37.3% done | ETA(epoch): 636s + [Epoch 186][20/33] loss=0.840197 avg=0.851712 VRAM=39.6GiB | 37.3% done | ETA(epoch): 360s + [Epoch 186][30/33] loss=0.765868 avg=0.848791 VRAM=39.9GiB | 37.4% done | ETA(epoch): 83s + Train loss: 0.848697 (912.7s) ETA: 5194min + Val loss: 0.848860 [t_0.0-0.2=1.0645 t_0.2-0.4=0.9987 t_0.4-0.6=0.8186 t_0.6-0.8=0.6868 t_0.8-1.0=0.6309] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0186 + Deleted old checkpoint: checkpoint_epoch_0183 +[MEM @ epoch 186 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 187/499 (37% done) --- + [Epoch 187][10/33] loss=0.759789 avg=0.835982 VRAM=39.9GiB | 37.5% done | ETA(epoch): 635s + [Epoch 187][20/33] loss=0.880692 avg=0.838104 VRAM=39.6GiB | 37.5% done | ETA(epoch): 359s + [Epoch 187][30/33] loss=0.749569 avg=0.851972 VRAM=39.9GiB | 37.6% done | ETA(epoch): 83s + Train loss: 0.846775 (912.4s) ETA: 5177min + Val loss: 0.808032 [t_0.0-0.2=1.0679 t_0.2-0.4=0.9908 t_0.4-0.6=0.8214 t_0.6-0.8=0.6980 t_0.8-1.0=0.6323] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0187 + Deleted old checkpoint: checkpoint_epoch_0184 +[MEM @ epoch 187 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 188/499 (38% done) --- + [Epoch 188][10/33] loss=0.903873 avg=0.855170 VRAM=39.9GiB | 37.7% done | ETA(epoch): 635s + [Epoch 188][20/33] loss=0.836850 avg=0.844160 VRAM=39.6GiB | 37.7% done | ETA(epoch): 359s + [Epoch 188][30/33] loss=0.867091 avg=0.845224 VRAM=39.9GiB | 37.8% done | ETA(epoch): 83s + Train loss: 0.844333 (912.3s) ETA: 5160min + Val loss: 0.870750 [t_0.0-0.2=1.0694 t_0.2-0.4=1.0046 t_0.4-0.6=0.8513 t_0.6-0.8=0.6997 t_0.8-1.0=0.6192] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0188 + Deleted old checkpoint: checkpoint_epoch_0185 +[MEM @ epoch 188 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 189/499 (38% done) --- + [Epoch 189][10/33] loss=0.899914 avg=0.865620 VRAM=39.9GiB | 37.9% done | ETA(epoch): 636s + [Epoch 189][20/33] loss=0.945024 avg=0.864224 VRAM=39.6GiB | 37.9% done | ETA(epoch): 360s + [Epoch 189][30/33] loss=0.760071 avg=0.846213 VRAM=39.9GiB | 38.0% done | ETA(epoch): 83s + Train loss: 0.850808 (913.0s) ETA: 5143min + Val loss: 0.820823 [t_0.0-0.2=1.0804 t_0.2-0.4=1.0175 t_0.4-0.6=0.8570 t_0.6-0.8=0.6723 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0189 + Deleted old checkpoint: checkpoint_epoch_0186 +[MEM @ epoch 189 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 190/499 (38% done) --- + [Epoch 190][10/33] loss=0.865993 avg=0.842535 VRAM=39.9GiB | 38.1% done | ETA(epoch): 636s + [Epoch 190][20/33] loss=0.891748 avg=0.858607 VRAM=39.6GiB | 38.1% done | ETA(epoch): 359s + [Epoch 190][30/33] loss=0.836999 avg=0.846304 VRAM=39.9GiB | 38.2% done | ETA(epoch): 83s + Train loss: 0.843819 (912.2s) ETA: 5126min + Val loss: 0.846190 [t_0.0-0.2=1.0424 t_0.2-0.4=0.9959 t_0.4-0.6=0.8334 t_0.6-0.8=0.7106 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0190 + Deleted old checkpoint: checkpoint_epoch_0187 +[MEM @ epoch 190 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 191/499 (38% done) --- + [Epoch 191][10/33] loss=0.906768 avg=0.841290 VRAM=39.9GiB | 38.3% done | ETA(epoch): 635s + [Epoch 191][20/33] loss=0.866782 avg=0.855980 VRAM=39.6GiB | 38.3% done | ETA(epoch): 359s + [Epoch 191][30/33] loss=0.891500 avg=0.851464 VRAM=39.9GiB | 38.4% done | ETA(epoch): 83s + Train loss: 0.851475 (912.3s) ETA: 5109min + Val loss: 0.854881 [t_0.0-0.2=1.0654 t_0.2-0.4=0.9896 t_0.4-0.6=0.8686 t_0.6-0.8=0.7029 t_0.8-1.0=0.6239] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0191 + Deleted old checkpoint: checkpoint_epoch_0188 +[MEM @ epoch 191 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 192/499 (38% done) --- + [Epoch 192][10/33] loss=0.854690 avg=0.853811 VRAM=39.9GiB | 38.5% done | ETA(epoch): 636s + [Epoch 192][20/33] loss=0.865423 avg=0.856741 VRAM=39.6GiB | 38.5% done | ETA(epoch): 360s + [Epoch 192][30/33] loss=0.778978 avg=0.851905 VRAM=39.9GiB | 38.6% done | ETA(epoch): 83s + Train loss: 0.851963 (912.5s) ETA: 5092min + Val loss: 0.843733 [t_0.0-0.2=1.0696 t_0.2-0.4=0.9900 t_0.4-0.6=0.8504 t_0.6-0.8=0.7246 t_0.8-1.0=0.6409] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0192 + Deleted old checkpoint: checkpoint_epoch_0189 +[MEM @ epoch 192 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 193/499 (39% done) --- + [Epoch 193][10/33] loss=0.857460 avg=0.854052 VRAM=39.9GiB | 38.7% done | ETA(epoch): 636s + [Epoch 193][20/33] loss=0.876554 avg=0.855294 VRAM=39.6GiB | 38.7% done | ETA(epoch): 359s + [Epoch 193][30/33] loss=0.782072 avg=0.858185 VRAM=39.9GiB | 38.8% done | ETA(epoch): 83s + Train loss: 0.862371 (912.6s) ETA: 5075min + Val loss: 0.846158 [t_0.0-0.2=1.0664 t_0.2-0.4=0.9959 t_0.4-0.6=0.8721 t_0.6-0.8=0.6912 t_0.8-1.0=0.6394] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0193 + Deleted old checkpoint: checkpoint_epoch_0190 +[MEM @ epoch 193 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 194/499 (39% done) --- + [Epoch 194][10/33] loss=0.871863 avg=0.847100 VRAM=39.9GiB | 38.9% done | ETA(epoch): 636s + [Epoch 194][20/33] loss=0.930483 avg=0.846795 VRAM=39.6GiB | 38.9% done | ETA(epoch): 359s + [Epoch 194][30/33] loss=0.820679 avg=0.856923 VRAM=39.9GiB | 39.0% done | ETA(epoch): 83s + Train loss: 0.859775 (912.5s) ETA: 5058min + Val loss: 0.827362 [t_0.0-0.2=1.0844 t_0.2-0.4=0.9825 t_0.4-0.6=0.8422 t_0.6-0.8=0.7168 t_0.8-1.0=0.6345] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0194 + Deleted old checkpoint: checkpoint_epoch_0191 +[MEM @ epoch 194 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 195/499 (39% done) --- + [Epoch 195][10/33] loss=0.832894 avg=0.871198 VRAM=39.9GiB | 39.1% done | ETA(epoch): 636s + [Epoch 195][20/33] loss=0.889590 avg=0.871954 VRAM=39.6GiB | 39.1% done | ETA(epoch): 359s + [Epoch 195][30/33] loss=0.896607 avg=0.858312 VRAM=39.9GiB | 39.2% done | ETA(epoch): 83s + Train loss: 0.853811 (912.5s) ETA: 5040min + Val loss: 0.843540 [t_0.0-0.2=1.0862 t_0.2-0.4=0.9834 t_0.4-0.6=0.8177 t_0.6-0.8=0.7044 t_0.8-1.0=0.6228] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0195 + Deleted old checkpoint: checkpoint_epoch_0192 +[MEM @ epoch 195 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 196/499 (39% done) --- + [Epoch 196][10/33] loss=0.762946 avg=0.844781 VRAM=39.9GiB | 39.3% done | ETA(epoch): 636s + [Epoch 196][20/33] loss=0.793707 avg=0.854470 VRAM=39.6GiB | 39.3% done | ETA(epoch): 359s + [Epoch 196][30/33] loss=0.857587 avg=0.863553 VRAM=39.9GiB | 39.4% done | ETA(epoch): 83s + Train loss: 0.859095 (912.2s) ETA: 5023min + Val loss: 0.846970 [t_0.0-0.2=1.0582 t_0.2-0.4=0.9992 t_0.4-0.6=0.8316 t_0.6-0.8=0.7092 t_0.8-1.0=0.6399] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0196 + Deleted old checkpoint: checkpoint_epoch_0193 +[MEM @ epoch 196 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 197/499 (39% done) --- + [Epoch 197][10/33] loss=0.823494 avg=0.840965 VRAM=39.9GiB | 39.5% done | ETA(epoch): 636s + [Epoch 197][20/33] loss=0.954612 avg=0.855743 VRAM=39.6GiB | 39.5% done | ETA(epoch): 359s + [Epoch 197][30/33] loss=0.824761 avg=0.854891 VRAM=39.9GiB | 39.6% done | ETA(epoch): 83s + Train loss: 0.848463 (912.7s) ETA: 5006min + Val loss: 0.866672 [t_0.0-0.2=1.0700 t_0.2-0.4=0.9921 t_0.4-0.6=0.8690 t_0.6-0.8=0.7148 t_0.8-1.0=0.6406] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0197 + Deleted old checkpoint: checkpoint_epoch_0194 +[MEM @ epoch 197 end] RAM: 18.8/188.4 GiB (10.0%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 198/499 (40% done) --- + [Epoch 198][10/33] loss=0.798976 avg=0.832042 VRAM=39.9GiB | 39.7% done | ETA(epoch): 636s + [Epoch 198][20/33] loss=0.777333 avg=0.822268 VRAM=39.6GiB | 39.7% done | ETA(epoch): 359s + [Epoch 198][30/33] loss=0.913563 avg=0.837428 VRAM=39.9GiB | 39.8% done | ETA(epoch): 83s + Train loss: 0.837183 (912.6s) ETA: 4990min + Val loss: 0.847259 [t_0.0-0.2=1.0740 t_0.2-0.4=0.9951 t_0.4-0.6=0.8480 t_0.6-0.8=0.7112 t_0.8-1.0=0.6358] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0198 + Deleted old checkpoint: checkpoint_epoch_0195 +[MEM @ epoch 198 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 199/499 (40% done) --- + [Epoch 199][10/33] loss=0.786212 avg=0.852494 VRAM=39.9GiB | 39.9% done | ETA(epoch): 636s + [Epoch 199][20/33] loss=0.781778 avg=0.851500 VRAM=39.6GiB | 39.9% done | ETA(epoch): 360s + [Epoch 199][30/33] loss=0.803492 avg=0.853064 VRAM=39.9GiB | 40.0% done | ETA(epoch): 83s + Train loss: 0.854592 (912.8s) ETA: 4973min + Val loss: 0.844270 [t_0.0-0.2=1.0720 t_0.2-0.4=0.9884 t_0.4-0.6=0.8642 t_0.6-0.8=0.6996 t_0.8-1.0=0.6227] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0199 + Deleted old checkpoint: checkpoint_epoch_0196 +[MEM @ epoch 199 end] RAM: 18.7/188.4 GiB (9.9%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 200/499 (40% done) --- + [MilestoneVis] train_0 step 6600 ✓ + [MilestoneGrid] train_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 6600 ✓ + [MilestoneGrid] train_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 6600 ✓ + [MilestoneGrid] val_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 6600 ✓ + [MilestoneGrid] val_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 6600 ✓ + [MilestoneGrid] test_0 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 6600 ✓ + [MilestoneGrid] test_1 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 6600 ✓ + [MilestoneGrid] test_2 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 6600 ✓ + [MilestoneGrid] test_3 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 6600 ✓ + [MilestoneGrid] test_4 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 6600 ✓ + [MilestoneGrid] test_5 → output/lora_512_dim_512_H200/milestone_vis/milestone_grid_test_5.png + [Epoch 200][10/33] loss=0.843996 avg=0.875490 VRAM=39.9GiB | 40.1% done | ETA(epoch): 2371s + [Epoch 200][20/33] loss=0.921360 avg=0.862111 VRAM=39.6GiB | 40.1% done | ETA(epoch): 850s + [Epoch 200][30/33] loss=0.797046 avg=0.858920 VRAM=39.9GiB | 40.2% done | ETA(epoch): 158s + Train loss: 0.860322 (1667.2s) ETA: 4974min + Val loss: 0.861254 [t_0.0-0.2=1.0748 t_0.2-0.4=0.9602 t_0.4-0.6=0.8851 t_0.6-0.8=0.7108 t_0.8-1.0=0.6380] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0200 + Deleted old checkpoint: checkpoint_epoch_0197 +[MEM @ epoch 200 end] RAM: 19.2/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 201/499 (40% done) --- + [Epoch 201][10/33] loss=0.800444 avg=0.853473 VRAM=39.9GiB | 40.3% done | ETA(epoch): 636s + [Epoch 201][20/33] loss=0.860110 avg=0.859221 VRAM=39.6GiB | 40.3% done | ETA(epoch): 359s + [Epoch 201][30/33] loss=0.786999 avg=0.863950 VRAM=39.9GiB | 40.4% done | ETA(epoch): 83s + Train loss: 0.862465 (912.4s) ETA: 4957min + Val loss: 0.854404 [t_0.0-0.2=1.0680 t_0.2-0.4=1.0171 t_0.4-0.6=0.8437 t_0.6-0.8=0.7193 t_0.8-1.0=0.6324] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0201 + Deleted old checkpoint: checkpoint_epoch_0198 +[MEM @ epoch 201 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 202/499 (40% done) --- + [Epoch 202][10/33] loss=0.864677 avg=0.849238 VRAM=39.9GiB | 40.5% done | ETA(epoch): 636s + [Epoch 202][20/33] loss=0.991960 avg=0.862171 VRAM=39.6GiB | 40.5% done | ETA(epoch): 360s + [Epoch 202][30/33] loss=0.853706 avg=0.870997 VRAM=39.9GiB | 40.6% done | ETA(epoch): 83s + Train loss: 0.874044 (912.6s) ETA: 4940min + Val loss: 0.820971 [t_0.0-0.2=1.0615 t_0.2-0.4=0.9830 t_0.4-0.6=0.8170 t_0.6-0.8=0.7142 t_0.8-1.0=0.6368] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0202 + Deleted old checkpoint: checkpoint_epoch_0199 +[MEM @ epoch 202 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 203/499 (41% done) --- + [Epoch 203][10/33] loss=0.941129 avg=0.831131 VRAM=39.9GiB | 40.7% done | ETA(epoch): 636s + [Epoch 203][20/33] loss=0.855521 avg=0.845549 VRAM=39.6GiB | 40.7% done | ETA(epoch): 359s + [Epoch 203][30/33] loss=0.807306 avg=0.833145 VRAM=39.9GiB | 40.8% done | ETA(epoch): 83s + Train loss: 0.836260 (912.1s) ETA: 4923min + Val loss: 0.857492 [t_0.0-0.2=1.0941 t_0.2-0.4=0.9803 t_0.4-0.6=0.8486 t_0.6-0.8=0.6679 t_0.8-1.0=0.6382] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0203 + Deleted old checkpoint: checkpoint_epoch_0200 +[MEM @ epoch 203 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 204/499 (41% done) --- + [Epoch 204][10/33] loss=0.883312 avg=0.853672 VRAM=39.9GiB | 40.9% done | ETA(epoch): 635s + [Epoch 204][20/33] loss=0.799054 avg=0.858644 VRAM=39.6GiB | 40.9% done | ETA(epoch): 359s + [Epoch 204][30/33] loss=0.818790 avg=0.860111 VRAM=39.9GiB | 41.0% done | ETA(epoch): 83s + Train loss: 0.863430 (911.9s) ETA: 4906min + Val loss: 0.815379 [t_0.0-0.2=1.0559 t_0.2-0.4=0.9833 t_0.4-0.6=0.8079 t_0.6-0.8=0.7129 t_0.8-1.0=0.6182] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0204 + Deleted old checkpoint: checkpoint_epoch_0201 +[MEM @ epoch 204 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 205/499 (41% done) --- + [Epoch 205][10/33] loss=0.906569 avg=0.841493 VRAM=39.9GiB | 41.1% done | ETA(epoch): 636s + [Epoch 205][20/33] loss=0.872024 avg=0.845386 VRAM=39.6GiB | 41.1% done | ETA(epoch): 359s + [Epoch 205][30/33] loss=0.882364 avg=0.850430 VRAM=39.9GiB | 41.2% done | ETA(epoch): 83s + Train loss: 0.852230 (912.7s) ETA: 4889min + Val loss: 0.858277 [t_0.0-0.2=1.0752 t_0.2-0.4=1.0223 t_0.4-0.6=0.8441 t_0.6-0.8=0.6949 t_0.8-1.0=0.6191] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0205 + Deleted old checkpoint: checkpoint_epoch_0202 +[MEM @ epoch 205 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 206/499 (41% done) --- + [Epoch 206][10/33] loss=0.926660 avg=0.868695 VRAM=39.9GiB | 41.3% done | ETA(epoch): 636s + [Epoch 206][20/33] loss=0.921834 avg=0.848345 VRAM=39.6GiB | 41.3% done | ETA(epoch): 360s + [Epoch 206][30/33] loss=0.824365 avg=0.850646 VRAM=39.9GiB | 41.4% done | ETA(epoch): 83s + Train loss: 0.851948 (912.9s) ETA: 4872min + Val loss: 0.862611 [t_0.0-0.2=1.0580 t_0.2-0.4=0.9965 t_0.4-0.6=0.8493 t_0.6-0.8=0.7301 t_0.8-1.0=0.6284] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0206 + Deleted old checkpoint: checkpoint_epoch_0203 +[MEM @ epoch 206 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 207/499 (41% done) --- + [Epoch 207][10/33] loss=0.857930 avg=0.833830 VRAM=39.9GiB | 41.5% done | ETA(epoch): 636s + [Epoch 207][20/33] loss=0.949626 avg=0.851326 VRAM=39.6GiB | 41.5% done | ETA(epoch): 360s + [Epoch 207][30/33] loss=0.902992 avg=0.849653 VRAM=39.9GiB | 41.6% done | ETA(epoch): 83s + Train loss: 0.846168 (912.9s) ETA: 4855min + Val loss: 0.845301 [t_0.0-0.2=1.0714 t_0.2-0.4=0.9760 t_0.4-0.6=0.8313 t_0.6-0.8=0.7151 t_0.8-1.0=0.6204] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0207 + Deleted old checkpoint: checkpoint_epoch_0204 +[MEM @ epoch 207 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 208/499 (42% done) --- + [Epoch 208][10/33] loss=0.908799 avg=0.869004 VRAM=39.9GiB | 41.7% done | ETA(epoch): 636s + [Epoch 208][20/33] loss=0.804385 avg=0.856073 VRAM=39.6GiB | 41.7% done | ETA(epoch): 360s + [Epoch 208][30/33] loss=0.774320 avg=0.850135 VRAM=39.9GiB | 41.8% done | ETA(epoch): 83s + Train loss: 0.850770 (913.0s) ETA: 4837min + Val loss: 0.826197 [t_0.0-0.2=1.0911 t_0.2-0.4=1.0101 t_0.4-0.6=0.8301 t_0.6-0.8=0.7121 t_0.8-1.0=0.6295] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0208 + Deleted old checkpoint: checkpoint_epoch_0205 +[MEM @ epoch 208 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 209/499 (42% done) --- + [Epoch 209][10/33] loss=0.856759 avg=0.830629 VRAM=39.9GiB | 41.9% done | ETA(epoch): 636s + [Epoch 209][20/33] loss=0.840570 avg=0.838980 VRAM=39.6GiB | 41.9% done | ETA(epoch): 360s + [Epoch 209][30/33] loss=0.816579 avg=0.838752 VRAM=39.9GiB | 42.0% done | ETA(epoch): 83s + Train loss: 0.838880 (913.0s) ETA: 4820min + Val loss: 0.878689 [t_0.0-0.2=1.0647 t_0.2-0.4=0.9804 t_0.4-0.6=0.8272 t_0.6-0.8=0.7008 t_0.8-1.0=0.6480] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0209 + Deleted old checkpoint: checkpoint_epoch_0206 +[MEM @ epoch 209 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 210/499 (42% done) --- + [Epoch 210][10/33] loss=0.808588 avg=0.852872 VRAM=39.9GiB | 42.1% done | ETA(epoch): 636s + [Epoch 210][20/33] loss=0.905188 avg=0.849408 VRAM=39.6GiB | 42.1% done | ETA(epoch): 360s + [Epoch 210][30/33] loss=0.910535 avg=0.842125 VRAM=39.9GiB | 42.2% done | ETA(epoch): 83s + Train loss: 0.847251 (913.4s) ETA: 4803min + Val loss: 0.852478 [t_0.0-0.2=1.0754 t_0.2-0.4=0.9995 t_0.4-0.6=0.8691 t_0.6-0.8=0.7318 t_0.8-1.0=0.6273] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0210 + Deleted old checkpoint: checkpoint_epoch_0207 +[MEM @ epoch 210 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 211/499 (42% done) --- + [Epoch 211][10/33] loss=0.768167 avg=0.828354 VRAM=39.9GiB | 42.3% done | ETA(epoch): 637s + [Epoch 211][20/33] loss=0.848949 avg=0.846871 VRAM=39.6GiB | 42.3% done | ETA(epoch): 360s + [Epoch 211][30/33] loss=0.819584 avg=0.847508 VRAM=39.9GiB | 42.4% done | ETA(epoch): 83s + Train loss: 0.847522 (913.5s) ETA: 4786min + Val loss: 0.854377 [t_0.0-0.2=1.0664 t_0.2-0.4=1.0343 t_0.4-0.6=0.8301 t_0.6-0.8=0.6945 t_0.8-1.0=0.6088] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0211 + Deleted old checkpoint: checkpoint_epoch_0208 +[MEM @ epoch 211 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 212/499 (42% done) --- + [Epoch 212][10/33] loss=0.787259 avg=0.848840 VRAM=39.9GiB | 42.5% done | ETA(epoch): 636s + [Epoch 212][20/33] loss=0.841573 avg=0.850989 VRAM=39.6GiB | 42.5% done | ETA(epoch): 360s + [Epoch 212][30/33] loss=0.907631 avg=0.854504 VRAM=39.9GiB | 42.6% done | ETA(epoch): 83s + Train loss: 0.853241 (912.9s) ETA: 4769min + Val loss: 0.809178 [t_0.0-0.2=1.0825 t_0.2-0.4=0.9853 t_0.4-0.6=0.8181 t_0.6-0.8=0.7165 t_0.8-1.0=0.6140] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0212 + Deleted old checkpoint: checkpoint_epoch_0209 +[MEM @ epoch 212 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 213/499 (43% done) --- + [Epoch 213][10/33] loss=0.862554 avg=0.838237 VRAM=39.9GiB | 42.7% done | ETA(epoch): 636s + [Epoch 213][20/33] loss=0.777605 avg=0.825144 VRAM=39.6GiB | 42.7% done | ETA(epoch): 360s + [Epoch 213][30/33] loss=0.833310 avg=0.833088 VRAM=39.9GiB | 42.8% done | ETA(epoch): 83s + Train loss: 0.837102 (913.0s) ETA: 4752min + Val loss: 0.845554 [t_0.0-0.2=1.0877 t_0.2-0.4=0.9818 t_0.4-0.6=0.8495 t_0.6-0.8=0.7207 t_0.8-1.0=0.6410] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0213 + Deleted old checkpoint: checkpoint_epoch_0210 +[MEM @ epoch 213 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 214/499 (43% done) --- + [Epoch 214][10/33] loss=0.862508 avg=0.844423 VRAM=39.9GiB | 42.9% done | ETA(epoch): 636s + [Epoch 214][20/33] loss=0.854492 avg=0.857577 VRAM=39.6GiB | 42.9% done | ETA(epoch): 360s + [Epoch 214][30/33] loss=0.759426 avg=0.842901 VRAM=39.9GiB | 43.0% done | ETA(epoch): 83s + Train loss: 0.840401 (912.5s) ETA: 4735min + Val loss: 0.865749 [t_0.0-0.2=1.0704 t_0.2-0.4=0.9961 t_0.4-0.6=0.8438 t_0.6-0.8=0.7252 t_0.8-1.0=0.6282] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0214 + Deleted old checkpoint: checkpoint_epoch_0211 +[MEM @ epoch 214 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 215/499 (43% done) --- + [Epoch 215][10/33] loss=0.839218 avg=0.862208 VRAM=39.9GiB | 43.1% done | ETA(epoch): 636s + [Epoch 215][20/33] loss=0.941796 avg=0.871676 VRAM=39.6GiB | 43.1% done | ETA(epoch): 359s + [Epoch 215][30/33] loss=0.895742 avg=0.860822 VRAM=39.9GiB | 43.2% done | ETA(epoch): 83s + Train loss: 0.861315 (912.5s) ETA: 4718min + Val loss: 0.847694 [t_0.0-0.2=1.0698 t_0.2-0.4=0.9802 t_0.4-0.6=0.8472 t_0.6-0.8=0.6945 t_0.8-1.0=0.6275] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0215 + Deleted old checkpoint: checkpoint_epoch_0212 +[MEM @ epoch 215 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 216/499 (43% done) --- + [Epoch 216][10/33] loss=0.844135 avg=0.827783 VRAM=39.9GiB | 43.3% done | ETA(epoch): 636s + [Epoch 216][20/33] loss=0.860500 avg=0.846758 VRAM=39.6GiB | 43.3% done | ETA(epoch): 359s + [Epoch 216][30/33] loss=0.802405 avg=0.846615 VRAM=39.9GiB | 43.4% done | ETA(epoch): 83s + Train loss: 0.845157 (912.1s) ETA: 4701min + Val loss: 0.878340 [t_0.0-0.2=1.0644 t_0.2-0.4=0.9976 t_0.4-0.6=0.8397 t_0.6-0.8=0.7409 t_0.8-1.0=0.6518] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0216 + Deleted old checkpoint: checkpoint_epoch_0213 +[MEM @ epoch 216 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 217/499 (43% done) --- + [Epoch 217][10/33] loss=0.879442 avg=0.848447 VRAM=39.9GiB | 43.5% done | ETA(epoch): 635s + [Epoch 217][20/33] loss=0.816535 avg=0.851184 VRAM=39.6GiB | 43.5% done | ETA(epoch): 359s + [Epoch 217][30/33] loss=0.854415 avg=0.845765 VRAM=39.9GiB | 43.6% done | ETA(epoch): 83s + Train loss: 0.847905 (911.5s) ETA: 4684min + Val loss: 0.860766 [t_0.0-0.2=1.0697 t_0.2-0.4=0.9869 t_0.4-0.6=0.8440 t_0.6-0.8=0.7280 t_0.8-1.0=0.6333] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0217 + Deleted old checkpoint: checkpoint_epoch_0214 +[MEM @ epoch 217 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 218/499 (44% done) --- + [Epoch 218][10/33] loss=0.834583 avg=0.837079 VRAM=39.9GiB | 43.7% done | ETA(epoch): 635s + [Epoch 218][20/33] loss=0.842316 avg=0.847915 VRAM=39.6GiB | 43.7% done | ETA(epoch): 359s + [Epoch 218][30/33] loss=0.905083 avg=0.849038 VRAM=39.9GiB | 43.8% done | ETA(epoch): 83s + Train loss: 0.847027 (911.5s) ETA: 4667min + Val loss: 0.876919 [t_0.0-0.2=1.0650 t_0.2-0.4=1.0069 t_0.4-0.6=0.8575 t_0.6-0.8=0.7190 t_0.8-1.0=0.6432] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0218 + Deleted old checkpoint: checkpoint_epoch_0215 +[MEM @ epoch 218 end] RAM: 19.3/188.4 GiB (10.2%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 219/499 (44% done) --- + [Epoch 219][10/33] loss=0.820577 avg=0.863852 VRAM=39.9GiB | 43.9% done | ETA(epoch): 635s + [Epoch 219][20/33] loss=0.853824 avg=0.842444 VRAM=39.6GiB | 43.9% done | ETA(epoch): 359s + [Epoch 219][30/33] loss=0.872728 avg=0.842902 VRAM=39.9GiB | 44.0% done | ETA(epoch): 83s + Train loss: 0.848538 (911.9s) ETA: 4650min + Val loss: 0.846330 [t_0.0-0.2=1.0341 t_0.2-0.4=0.9907 t_0.4-0.6=0.8748 t_0.6-0.8=0.7047 t_0.8-1.0=0.6373] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0219 + Deleted old checkpoint: checkpoint_epoch_0216 +[MEM @ epoch 219 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 220/499 (44% done) --- + [Epoch 220][10/33] loss=0.928052 avg=0.853906 VRAM=39.9GiB | 44.1% done | ETA(epoch): 636s + [Epoch 220][20/33] loss=0.852985 avg=0.842315 VRAM=39.6GiB | 44.1% done | ETA(epoch): 359s + [Epoch 220][30/33] loss=0.840721 avg=0.840693 VRAM=39.9GiB | 44.2% done | ETA(epoch): 83s + Train loss: 0.840898 (912.3s) ETA: 4633min + Val loss: 0.855959 [t_0.0-0.2=1.0794 t_0.2-0.4=1.0040 t_0.4-0.6=0.8256 t_0.6-0.8=0.7125 t_0.8-1.0=0.6397] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0220 + Deleted old checkpoint: checkpoint_epoch_0217 +[MEM @ epoch 220 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 221/499 (44% done) --- + [Epoch 221][10/33] loss=0.892510 avg=0.864788 VRAM=39.9GiB | 44.3% done | ETA(epoch): 636s + [Epoch 221][20/33] loss=0.828239 avg=0.854485 VRAM=39.6GiB | 44.3% done | ETA(epoch): 359s + [Epoch 221][30/33] loss=0.924230 avg=0.849400 VRAM=39.9GiB | 44.4% done | ETA(epoch): 83s + Train loss: 0.850441 (912.4s) ETA: 4616min + Val loss: 0.837619 [t_0.0-0.2=1.0607 t_0.2-0.4=1.0139 t_0.4-0.6=0.8649 t_0.6-0.8=0.7033 t_0.8-1.0=0.6392] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0221 + Deleted old checkpoint: checkpoint_epoch_0218 +[MEM @ epoch 221 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 222/499 (44% done) --- + [Epoch 222][10/33] loss=0.871085 avg=0.814070 VRAM=39.9GiB | 44.5% done | ETA(epoch): 636s + [Epoch 222][20/33] loss=0.810849 avg=0.830843 VRAM=39.6GiB | 44.5% done | ETA(epoch): 359s + [Epoch 222][30/33] loss=0.829554 avg=0.842741 VRAM=39.9GiB | 44.6% done | ETA(epoch): 83s + Train loss: 0.844204 (912.7s) ETA: 4599min + Val loss: 0.888340 [t_0.0-0.2=1.0593 t_0.2-0.4=0.9847 t_0.4-0.6=0.8523 t_0.6-0.8=0.7365 t_0.8-1.0=0.6242] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0222 + Deleted old checkpoint: checkpoint_epoch_0219 +[MEM @ epoch 222 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 223/499 (45% done) --- + [Epoch 223][10/33] loss=0.875202 avg=0.864271 VRAM=39.9GiB | 44.7% done | ETA(epoch): 636s + [Epoch 223][20/33] loss=0.929473 avg=0.859538 VRAM=39.6GiB | 44.7% done | ETA(epoch): 360s + [Epoch 223][30/33] loss=0.918701 avg=0.864859 VRAM=39.9GiB | 44.8% done | ETA(epoch): 83s + Train loss: 0.861639 (912.7s) ETA: 4582min + Val loss: 0.845808 [t_0.0-0.2=1.0806 t_0.2-0.4=1.0114 t_0.4-0.6=0.8403 t_0.6-0.8=0.7086 t_0.8-1.0=0.6337] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0223 + Deleted old checkpoint: checkpoint_epoch_0220 +[MEM @ epoch 223 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 224/499 (45% done) --- + [Epoch 224][10/33] loss=0.849347 avg=0.842846 VRAM=39.9GiB | 44.9% done | ETA(epoch): 636s + [Epoch 224][20/33] loss=0.810124 avg=0.850665 VRAM=39.6GiB | 44.9% done | ETA(epoch): 359s + [Epoch 224][30/33] loss=0.865482 avg=0.846467 VRAM=39.9GiB | 45.0% done | ETA(epoch): 83s + Train loss: 0.849033 (912.3s) ETA: 4565min + Val loss: 0.875091 [t_0.0-0.2=1.0678 t_0.2-0.4=1.0042 t_0.4-0.6=0.8327 t_0.6-0.8=0.7100 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0224 + Deleted old checkpoint: checkpoint_epoch_0221 +[MEM @ epoch 224 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 225/499 (45% done) --- + [Epoch 225][10/33] loss=0.965634 avg=0.840170 VRAM=39.9GiB | 45.1% done | ETA(epoch): 636s + [Epoch 225][20/33] loss=0.901790 avg=0.849529 VRAM=39.6GiB | 45.1% done | ETA(epoch): 359s + [Epoch 225][30/33] loss=0.849927 avg=0.844946 VRAM=39.9GiB | 45.2% done | ETA(epoch): 83s + Train loss: 0.848355 (912.4s) ETA: 4548min + Val loss: 0.856105 [t_0.0-0.2=1.0726 t_0.2-0.4=0.9854 t_0.4-0.6=0.8549 t_0.6-0.8=0.6990 t_0.8-1.0=0.6237] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0225 + Deleted old checkpoint: checkpoint_epoch_0222 +[MEM @ epoch 225 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 226/499 (45% done) --- + [Epoch 226][10/33] loss=0.951162 avg=0.890607 VRAM=39.9GiB | 45.3% done | ETA(epoch): 636s + [Epoch 226][20/33] loss=0.917932 avg=0.861690 VRAM=39.6GiB | 45.3% done | ETA(epoch): 360s + [Epoch 226][30/33] loss=0.792541 avg=0.855458 VRAM=39.9GiB | 45.4% done | ETA(epoch): 83s + Train loss: 0.854443 (912.9s) ETA: 4531min + Val loss: 0.820153 [t_0.0-0.2=1.0540 t_0.2-0.4=0.9727 t_0.4-0.6=0.8283 t_0.6-0.8=0.7143 t_0.8-1.0=0.6319] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0226 + Deleted old checkpoint: checkpoint_epoch_0223 +[MEM @ epoch 226 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 227/499 (45% done) --- + [Epoch 227][10/33] loss=0.907658 avg=0.852800 VRAM=39.9GiB | 45.5% done | ETA(epoch): 637s + [Epoch 227][20/33] loss=0.900784 avg=0.863259 VRAM=39.6GiB | 45.5% done | ETA(epoch): 360s + [Epoch 227][30/33] loss=0.770308 avg=0.848687 VRAM=39.9GiB | 45.6% done | ETA(epoch): 83s + Train loss: 0.847335 (913.0s) ETA: 4514min + Val loss: 0.839328 [t_0.0-0.2=1.0779 t_0.2-0.4=0.9756 t_0.4-0.6=0.8131 t_0.6-0.8=0.7139 t_0.8-1.0=0.6266] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0227 + Deleted old checkpoint: checkpoint_epoch_0224 +[MEM @ epoch 227 end] RAM: 19.5/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 228/499 (46% done) --- + [Epoch 228][10/33] loss=0.835625 avg=0.876083 VRAM=39.9GiB | 45.7% done | ETA(epoch): 636s + [Epoch 228][20/33] loss=0.844439 avg=0.854139 VRAM=39.6GiB | 45.7% done | ETA(epoch): 360s + [Epoch 228][30/33] loss=0.904590 avg=0.854596 VRAM=39.9GiB | 45.8% done | ETA(epoch): 83s + Train loss: 0.858684 (912.7s) ETA: 4498min + Val loss: 0.836368 [t_0.0-0.2=1.0687 t_0.2-0.4=0.9531 t_0.4-0.6=0.8371 t_0.6-0.8=0.7142 t_0.8-1.0=0.6338] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0228 + Deleted old checkpoint: checkpoint_epoch_0225 +[MEM @ epoch 228 end] RAM: 19.4/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 229/499 (46% done) --- + [Epoch 229][10/33] loss=0.785885 avg=0.850395 VRAM=39.9GiB | 45.9% done | ETA(epoch): 636s + [Epoch 229][20/33] loss=0.893034 avg=0.865443 VRAM=39.6GiB | 45.9% done | ETA(epoch): 360s + [Epoch 229][30/33] loss=0.871400 avg=0.848313 VRAM=39.9GiB | 46.0% done | ETA(epoch): 83s + Train loss: 0.844515 (913.1s) ETA: 4481min + Val loss: 0.867872 [t_0.0-0.2=1.0707 t_0.2-0.4=1.0001 t_0.4-0.6=0.8711 t_0.6-0.8=0.6901 t_0.8-1.0=0.6344] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0229 + Deleted old checkpoint: checkpoint_epoch_0226 +[MEM @ epoch 229 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 230/499 (46% done) --- + [Epoch 230][10/33] loss=0.796332 avg=0.854519 VRAM=39.9GiB | 46.1% done | ETA(epoch): 637s + [Epoch 230][20/33] loss=0.872610 avg=0.850738 VRAM=39.6GiB | 46.1% done | ETA(epoch): 360s + [Epoch 230][30/33] loss=0.859859 avg=0.841169 VRAM=39.9GiB | 46.2% done | ETA(epoch): 83s + Train loss: 0.840513 (913.1s) ETA: 4464min + Val loss: 0.866727 [t_0.0-0.2=1.0778 t_0.2-0.4=0.9754 t_0.4-0.6=0.8260 t_0.6-0.8=0.6861 t_0.8-1.0=0.6214] + Checkpoint saved: output/lora_512_dim_512_H200/checkpoint_epoch_0230 + Deleted old checkpoint: checkpoint_epoch_0227 +[MEM @ epoch 230 end] RAM: 19.3/188.4 GiB (10.3%) | VRAM: 38.9/139.8 GiB (27.8%) + +--- Epoch 231/499 (46% done) --- + [Epoch 231][10/33] loss=0.924717 avg=0.847905 VRAM=39.9GiB | 46.3% done | ETA(epoch): 636s + [Epoch 231][20/33] loss=0.846037 avg=0.851259 VRAM=39.6GiB | 46.3% done | ETA(epoch): 360s + [Epoch 231][30/33] loss=0.901988 avg=0.850571 VRAM=39.9GiB | 46.4% done | ETA(epoch): 83s