| 2025-10-24 00:38:25 - [1m[Sana][0m - INFO - Distributed environment: MULTI_GPU Backend: nccl | |
| Num processes: 8 | |
| Process index: 0 | |
| Local process index: 0 | |
| Device: cuda:0 | |
| Mixed precision type: bf16 | |
| 2025-10-24 00:38:25 - [1m[Sana][0m - INFO - Config: | |
| { | |
| "data": { | |
| "data_dir": [ | |
| "/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Datasets/IR" | |
| ], | |
| "caption_proportion": { | |
| "prompt": 1 | |
| }, | |
| "external_caption_suffixes": [], | |
| "external_clipscore_suffixes": [], | |
| "clip_thr_temperature": 0.1, | |
| "clip_thr": 25.0, | |
| "del_img_clip_thr": 0.0, | |
| "sort_dataset": false, | |
| "load_text_feat": false, | |
| "load_vae_feat": false, | |
| "transform": "default_train", | |
| "type": "IRImgDataset", | |
| "image_size": 256, | |
| "hq_only": false, | |
| "valid_num": 0, | |
| "data": null, | |
| "extra": null, | |
| "dset": "train_brief", | |
| "max_samples": null, | |
| "use_tokens": "ALL", | |
| "max_tokens": 400, | |
| "caption_type": "question", | |
| "return_meta": true, | |
| "need_resize": true, | |
| "synthetic_composite": true | |
| }, | |
| "model": { | |
| "model": "SD35M_D2C", | |
| "model_pretrained": "./checkpoints/stable-diffusion-3.5-medium/sd3.5_medium.safetensors", | |
| "shift": 3.0, | |
| "teacher": null, | |
| "input_channel": 16, | |
| "image_size": 256, | |
| "mixed_precision": "bf16", | |
| "fp32_attention": true, | |
| "load_from": "/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/IR/sd35_pix2pix/output/sd35m_d2c_breif/checkpoints/epoch_100_step_302601.pth", | |
| "discriminator_model": null, | |
| "teacher_model": null, | |
| "teacher_model_weight_dtype": null, | |
| "resume_from": null, | |
| "aspect_ratio_type": "ASPECT_RATIO_1024", | |
| "multi_scale": false, | |
| "pe_interpolation": 1.0, | |
| "micro_condition": false, | |
| "attn_type": "linear", | |
| "autocast_linear_attn": false, | |
| "ffn_type": "glumbconv", | |
| "mlp_acts": [ | |
| "silu", | |
| "silu", | |
| null | |
| ], | |
| "mlp_ratio": 2.5, | |
| "use_pe": false, | |
| "pos_embed_type": "sincos", | |
| "qk_norm": false, | |
| "class_dropout_prob": 0.1, | |
| "linear_head_dim": 32, | |
| "cross_norm": false, | |
| "cross_attn_type": "flash", | |
| "logvar": false, | |
| "cfg_scale": 4, | |
| "cfg_embed": false, | |
| "cfg_embed_scale": 1.0, | |
| "guidance_type": "classifier-free", | |
| "pag_applied_layers": [ | |
| 8 | |
| ], | |
| "ladd_multi_scale": true, | |
| "head_block_ids": null, | |
| "extra": null | |
| }, | |
| "vae": { | |
| "vae_type": "SDVAE", | |
| "vae_pretrained": "./checkpoints/stable-diffusion-3.5-medium/sd3.5_medium.safetensors", | |
| "weight_dtype": "float32", | |
| "scale_factor": 0.41407, | |
| "vae_latent_dim": 16, | |
| "vae_downsample_rate": 8, | |
| "sample_posterior": true, | |
| "extra": null | |
| }, | |
| "text_encoder": { | |
| "text_encoder_name": "empty", | |
| "text_encoder_pretrained": "./checkpoints/stable-diffusion-3.5-medium/text_encoders", | |
| "caption_channels": 4096, | |
| "y_norm": true, | |
| "y_norm_scale_factor": 0.01, | |
| "model_max_length": 300, | |
| "chi_prompt": [ | |
| "a photo of a cat", | |
| "Convenience store entrance at night. On the glass door, a vinyl decal reads 'OPEN FOR QUALITY'. Inside, shelves and fluorescent lights; outside, a cyclist passing by", | |
| "Sunrise beach, shallow tide washing over smooth sand. A piece of weathered driftwood lies near the shoreline with a subtle branded text [SOS] on its surface; wet sand reflections, micro-ripples, sun flare at horizon." | |
| ], | |
| "extra": null | |
| }, | |
| "connector": { | |
| "model": "QFormer", | |
| "model_pretrained": "None", | |
| "input_channel": 4096, | |
| "num_input_tokens": 400, | |
| "num_query_tokens": 77, | |
| "hidden_dim": 1024, | |
| "layers": 8, | |
| "heads": 16, | |
| "resume_from": "None", | |
| "load_from": "/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/IQAIR/sd35_conditioning_s1/output/sd35m_p2p_breif_connector/checkpoints/epoch_100_step_19001.pth" | |
| }, | |
| "assessment": { | |
| "model": "SDQA", | |
| "model_config": "./iqa/config.yaml" | |
| }, | |
| "scheduler": { | |
| "train_sampling_steps": 1000, | |
| "predict_flow_v": true, | |
| "noise_schedule": "linear_flow", | |
| "pred_sigma": false, | |
| "learn_sigma": true, | |
| "vis_sampler": "flow_dpm-solver", | |
| "flow_shift": 3.0, | |
| "weighting_scheme": "logit_normal", | |
| "weighting_scheme_discriminator": "logit_normal_trigflow", | |
| "add_noise_timesteps": [ | |
| 1.5708 | |
| ], | |
| "logit_mean": 0.0, | |
| "logit_std": 1.0, | |
| "logit_mean_discriminator": 0.0, | |
| "logit_std_discriminator": 1.0, | |
| "sigma_data": 0.5, | |
| "timestep_norm_scale_factor": 1.0, | |
| "extra": null, | |
| "recur_round": 4, | |
| "recur_truncated": false, | |
| "recur_loss_weights": [ | |
| 0.2, | |
| 0.4, | |
| 0.6, | |
| 1.0 | |
| ], | |
| "update_image": true | |
| }, | |
| "train": { | |
| "train_type": "bf16", | |
| "resume_from": "None", | |
| "num_workers": 10, | |
| "seed": 1229, | |
| "train_batch_size": 1, | |
| "num_epochs": 100, | |
| "gradient_accumulation_steps": 16, | |
| "grad_checkpointing": true, | |
| "gradient_clip": 0.1, | |
| "gc_step": 1, | |
| "optimizer": { | |
| "betas": [ | |
| 0.9, | |
| 0.999, | |
| 0.9999 | |
| ], | |
| "eps": [ | |
| 1e-30, | |
| 1e-16 | |
| ], | |
| "lr": 2e-06, | |
| "type": "CAMEWrapper", | |
| "weight_decay": 0.0 | |
| }, | |
| "optimizer_D": { | |
| "eps": 1e-10, | |
| "lr": 0.0001, | |
| "type": "AdamW", | |
| "weight_decay": 0.03 | |
| }, | |
| "load_from_optimizer": false, | |
| "load_from_lr_scheduler": false, | |
| "resume_lr_scheduler": true, | |
| "lr_schedule": "cosine", | |
| "lr_schedule_args": { | |
| "num_warmup_steps": 2000 | |
| }, | |
| "auto_lr": { | |
| "rule": "sqrt" | |
| }, | |
| "eval_batch_size": 16, | |
| "use_fsdp": false, | |
| "use_flash_attn": false, | |
| "eval_sampling_steps": 1000, | |
| "lora_rank": 4, | |
| "log_interval": 100, | |
| "mask_type": "null", | |
| "mask_loss_coef": 0.0, | |
| "load_mask_index": false, | |
| "snr_loss": false, | |
| "real_prompt_ratio": 1.0, | |
| "early_stop_hours": 10000.0, | |
| "save_image_epochs": 1, | |
| "save_model_epochs": 1, | |
| "save_model_steps": 5000, | |
| "visualize": true, | |
| "null_embed_root": "output/pretrained_models/", | |
| "valid_prompt_embed_root": "output/tmp_embed/", | |
| "validation_prompts": [ | |
| "dog", | |
| "portrait photo of a girl, photograph, highly detailed face, depth of field", | |
| "Self-portrait oil painting, a beautiful cyborg with golden hair, 8k", | |
| "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", | |
| "A photo of beautiful mountain with realistic sunset and blue lake, highly detailed, masterpiece" | |
| ], | |
| "local_save_vis": true, | |
| "deterministic_validation": true, | |
| "online_metric": false, | |
| "eval_metric_step": 2000, | |
| "online_metric_dir": "metric_helper", | |
| "work_dir": "output_c2c/sd35m_d2c_multi", | |
| "skip_step": 0, | |
| "loss_type": "huber", | |
| "huber_c": 0.001, | |
| "num_ddim_timesteps": 50, | |
| "ema_decay": 0.95, | |
| "debug_nan": false, | |
| "ema_update": false, | |
| "ema_rate": 0.9999, | |
| "tangent_warmup_steps": 10000, | |
| "scm_cfg_scale": [ | |
| 1.0 | |
| ], | |
| "cfg_interval": null, | |
| "scm_logvar_loss": true, | |
| "norm_invariant_to_spatial_dim": true, | |
| "norm_same_as_512_scale": false, | |
| "g_norm_constant": 0.1, | |
| "g_norm_r": 1.0, | |
| "show_gradient": false, | |
| "lr_scale": null, | |
| "adv_lambda": 1.0, | |
| "scm_loss": true, | |
| "scm_lambda": 1.0, | |
| "loss_scale": 1.0, | |
| "r1_penalty": false, | |
| "r1_penalty_weight": 1e-05, | |
| "diff_timesteps_D": true, | |
| "suffix_checkpoints": "disc", | |
| "misaligned_pairs_D": false, | |
| "discriminator_loss": "cross entropy", | |
| "largest_timestep": 1.5708, | |
| "train_largest_timestep": false, | |
| "largest_timestep_prob": 0.5, | |
| "extra": null | |
| }, | |
| "controlnet": null, | |
| "model_growth": null, | |
| "work_dir": "output_composite/sd35m_d2c_multi_bf16_lr2e6", | |
| "resume_from": null, | |
| "load_from": null, | |
| "debug": true, | |
| "caching": false, | |
| "report_to": "tensorboard", | |
| "tracker_project_name": "sana-baseline", | |
| "name": "tmp", | |
| "loss_report_name": "loss" | |
| } | |
| 2025-10-24 00:38:25 - [1m[Sana][0m - INFO - World_size: 8, seed: 1229 | |
| 2025-10-24 00:38:25 - [1m[Sana][0m - INFO - Initializing: DDP for training | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - VAE type: SDVAE, path: ./checkpoints/stable-diffusion-3.5-medium/sd3.5_medium.safetensors, weight_dtype: torch.float32 | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - VAE Params: 83.819683 M, dtype: torch.float32 | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - Trainable Params: [] | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - text_encoder type: empty, path: ./checkpoints/stable-diffusion-3.5-medium/text_encoders | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:38:30 - [1m[Sana][0m - INFO - DiT type: SD35M_D2C, path: ./checkpoints/stable-diffusion-3.5-medium/sd3.5_medium.safetensors, weight_dtype: bf16 | |
| 2025-10-24 00:38:36 - [1m[Sana][0m - INFO - Loading Pre-trained Weight for SD35M-D2C: /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/IR/sd35_pix2pix/output/sd35m_d2c_breif/checkpoints/epoch_100_step_302601.pth | |
| 2025-10-24 00:39:05 - [1m[Sana][0m - WARNING - Missing keys: [] | |
| 2025-10-24 00:39:05 - [1m[Sana][0m - WARNING - Unexpected keys: [] | |
| 2025-10-24 00:39:05 - [1m[Sana][0m - INFO - DiT Params: 2243.17152 M, dtype: torch.bfloat16 | |
| 2025-10-24 00:39:05 - [1m[Sana][0m - INFO - Trainable Params: ['context_embedder.bias', 'context_embedder.weight', 'final_layer.adaLN_modulation', 'final_layer.linear', 'joint_blocks.0', 'joint_blocks.1', 'joint_blocks.10', 'joint_blocks.11', 'joint_blocks.12', 'joint_blocks.13', 'joint_blocks.14', 'joint_blocks.15', 'joint_blocks.16', 'joint_blocks.17', 'joint_blocks.18', 'joint_blocks.19', 'joint_blocks.2', 'joint_blocks.20', 'joint_blocks.21', 'joint_blocks.22', 'joint_blocks.23', 'joint_blocks.3', 'joint_blocks.4', 'joint_blocks.5', 'joint_blocks.6', 'joint_blocks.7', 'joint_blocks.8', 'joint_blocks.9', 't_embedder.mlp', 'x_embedder.proj', 'y_embedder.mlp'] | |
| 2025-10-24 00:39:05 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:39:05 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:39:05 - [1m[Sana][0m - INFO - Connector type: QFormer, path: None, weight_dtype: torch.float32 | |
| 2025-10-24 00:39:15 - [1m[Sana][0m - INFO - Loading Pre-trained Weight for Connector: /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/IQAIR/sd35_conditioning_s1/output/sd35m_p2p_breif_connector/checkpoints/epoch_100_step_19001.pth | |
| 2025-10-24 00:39:18 - [1m[Sana][0m - WARNING - Missing keys: [] | |
| 2025-10-24 00:39:18 - [1m[Sana][0m - WARNING - Unexpected keys: [] | |
| 2025-10-24 00:39:21 - [1m[Sana][0m - INFO - Connector Params: 168.691715 M, dtype: torch.bfloat16 | |
| 2025-10-24 00:39:21 - [1m[Sana][0m - INFO - Trainable Params: ['clip_g_line', 'clip_g_norm.bias', 'clip_g_norm.weight', 'clip_g_pool.attn', 'clip_g_pool.query_token', 'clip_g_proj.fc1', 'clip_g_proj.fc2', 'clip_g_proj.gamma', 'clip_g_proj.ln', 'clip_l_line', 'clip_l_norm.bias', 'clip_l_norm.weight', 'clip_l_pool.attn', 'clip_l_pool.query_token', 'clip_l_proj.fc1', 'clip_l_proj.fc2', 'clip_l_proj.gamma', 'clip_l_proj.ln', 'input_norm.bias', 'input_norm.weight', 'input_proj.bias', 'input_proj.weight', 'query_tokens', 't5_xxl_proj.fc1', 't5_xxl_proj.fc2', 't5_xxl_proj.gamma', 't5_xxl_proj.ln', 'transformer.0', 'transformer.1', 'transformer.2', 'transformer.3', 'transformer.4', 'transformer.5', 'transformer.6', 'transformer.7'] | |
| 2025-10-24 00:39:21 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:39:21 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:39:21 - [1m[Sana][0m - INFO - IQA type: SDQA, config: ./iqa/config.yaml | |
| 2025-10-24 00:41:01 - [1m[Sana][0m - INFO - [!] Load pretrained delta ckpt from /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/ModelZoo/DepictQA/DepictQA2-DQ495K/ckpt.pt | |
| 2025-10-24 00:41:04 - [1m[Sana][0m - INFO - [!] Load pretrained delta ckpt from /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/IQA/SDQA/experiments/SD35/log_sd35m_s1/former_bf/ckpt/ckpt_epoch50.pt | |
| 2025-10-24 00:41:20 - [1m[Sana][0m - INFO - [!] Load pretrained abstractor and projector ckpt from /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/IQA/SDQA/experiments/SD35/log_sd35m_s2/former_bf/ckpt/ckpt_epoch1.pt | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - IQA Params: 6915.601408 M, dtype: torch.bfloat16 | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - Trainable Params: [] | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - Summary: IQAIR | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - Trainable Params Layers: ['connector.clip_g_line', 'connector.clip_g_norm.bias', 'connector.clip_g_norm.weight', 'connector.clip_g_pool.attn', 'connector.clip_g_pool.query_token', 'connector.clip_g_proj.fc1', 'connector.clip_g_proj.fc2', 'connector.clip_g_proj.gamma', 'connector.clip_g_proj.ln', 'connector.clip_l_line', 'connector.clip_l_norm.bias', 'connector.clip_l_norm.weight', 'connector.clip_l_pool.attn', 'connector.clip_l_pool.query_token', 'connector.clip_l_proj.fc1', 'connector.clip_l_proj.fc2', 'connector.clip_l_proj.gamma', 'connector.clip_l_proj.ln', 'connector.input_norm.bias', 'connector.input_norm.weight', 'connector.input_proj.bias', 'connector.input_proj.weight', 'connector.query_tokens', 'connector.t5_xxl_proj.fc1', 'connector.t5_xxl_proj.fc2', 'connector.t5_xxl_proj.gamma', 'connector.t5_xxl_proj.ln', 'connector.transformer.0', 'connector.transformer.1', 'connector.transformer.2', 'connector.transformer.3', 'connector.transformer.4', 'connector.transformer.5', 'connector.transformer.6', 'connector.transformer.7', 'model.context_embedder.bias', 'model.context_embedder.weight', 'model.final_layer.adaLN_modulation', 'model.final_layer.linear', 'model.joint_blocks.0', 'model.joint_blocks.1', 'model.joint_blocks.10', 'model.joint_blocks.11', 'model.joint_blocks.12', 'model.joint_blocks.13', 'model.joint_blocks.14', 'model.joint_blocks.15', 'model.joint_blocks.16', 'model.joint_blocks.17', 'model.joint_blocks.18', 'model.joint_blocks.19', 'model.joint_blocks.2', 'model.joint_blocks.20', 'model.joint_blocks.21', 'model.joint_blocks.22', 'model.joint_blocks.23', 'model.joint_blocks.3', 'model.joint_blocks.4', 'model.joint_blocks.5', 'model.joint_blocks.6', 'model.joint_blocks.7', 'model.joint_blocks.8', 'model.joint_blocks.9', 'model.t_embedder.mlp', 'model.x_embedder.proj', 'model.y_embedder.mlp'] | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - Trainable Params: 2411.863M, all params: 9327.465M, trainable%: 25.8577% | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - flow-prediction: True, noise schedule: linear_flow, flow shift: 3.0, flow weighting: logit_normal, logit-mean: 0.0, logit-std: 1.0 | |
| 2025-10-24 00:41:52 - [1m[Sana][0m - INFO - Constructing dataset IRImgDataset... | |
| 2025-10-24 00:41:54 - [1m[Sana][0m - INFO - [1m[31mUse Image Restoration Dataset, number of samples: 96804[0m | |
| 2025-10-24 00:41:54 - [1m[Sana][0m - INFO - Text max token length: 300 | |
| 2025-10-24 00:41:54 - [1m[Sana][0m - INFO - [1m[32mDataset IRImgDataset constructed: [0mtime: 1.72 s, length (use/ori): 96804/96804 | |
| 2025-10-24 00:42:02 - [1m[Sana][0m - INFO - Automatically adapt lr to 0.00000 (using sqrt scaling rule). | |
| 2025-10-24 00:42:03 - [1m[Sana][0m - INFO - CAMEWrapper Optimizer: total 1548 param groups, [32m844 are learnable, 704 are fix[0m. Lr group: 844 params with lr 0.00000; Weight decay group: 844 params with weight decay 0.0. | |
| 2025-10-24 00:42:03 - [1m[Sana][0m - INFO - Lr schedule: cosine, num_warmup_steps:16000. | |
| 2025-10-24 00:42:03 - [1m[Sana][0m - WARNING - [1m[32mBasic Setting: [0mlr: 0.00000, bs: 1, gc: True, gc_accum_step: 16, qk norm: False, fp32 attn: True, attn type: linear, ffn type: glumbconv, text encoder: empty, captions: {'prompt': 1}, precision: bf16 | |
| 2025-10-24 00:42:03 - [1m[Sana][0m - INFO - Set seed: 0 | |
| 2025-10-24 00:42:03 - [1m[Sana][0m - INFO - ############################################################## | |
| 2025-10-24 00:43:27 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1 | Local Step: 1 // 12101, total_eta: 1174 days, 6:16:18, epoch_eta:11 days, 17:47:59, time: all:0.838, data:0.041, vae:0.006, IQA:0.087, Connector:0.031, model:0.403, lr:0.000e+00, loss:0.326loss:0.2734 | |
| 2025-10-24 00:54:43 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 100 | Local Step: 100 // 12101, total_eta: 106 days, 8:48:19, epoch_eta:1 day, 1:19:09, time: all:6.757, data:0.001, vae:0.019, IQA:4.319, Connector:0.088, model:1.715, lr:4.243e-09, loss:0.139loss:0.3340, grad_norm:0.9142 | |
| 2025-10-24 01:04:47 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 200 | Local Step: 200 // 12101, total_eta: 95 days, 11:28:30, epoch_eta:22:32:23, time: all:6.041, data:0.001, vae:0.018, IQA:4.360, Connector:0.068, model:1.185, lr:8.485e-09, loss:0.180loss:0.3381, grad_norm:0.8320 | |
| 2025-10-24 01:14:54 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 300 | Local Step: 300 // 12101, total_eta: 91 days, 22:55:03, epoch_eta:21:31:38, time: all:6.065, data:0.001, vae:0.018, IQA:4.335, Connector:0.067, model:1.236, lr:1.273e-08, loss:0.127loss:0.3249, grad_norm:0.4899 | |
| 2025-10-24 01:25:00 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 400 | Local Step: 400 // 12101, total_eta: 90 days, 4:46:23, epoch_eta:20:56:20, time: all:6.068, data:0.001, vae:0.018, IQA:4.346, Connector:0.070, model:1.228, lr:1.768e-08, loss:0.727loss:0.3155, grad_norm:0.6812 | |
| 2025-10-24 01:35:04 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 500 | Local Step: 500 // 12101, total_eta: 89 days, 1:35:34, epoch_eta:20:30:04, time: all:6.040, data:0.001, vae:0.018, IQA:4.304, Connector:0.069, model:1.250, lr:2.192e-08, loss:0.010loss:0.3271, grad_norm:0.5016 | |
| 2025-10-24 01:45:08 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 600 | Local Step: 600 // 12101, total_eta: 88 days, 7:26:46, epoch_eta:20:09:12, time: all:6.041, data:0.001, vae:0.018, IQA:4.331, Connector:0.064, model:1.224, lr:2.616e-08, loss:0.031loss:0.3110, grad_norm:0.8615 | |
| 2025-10-24 01:55:08 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 700 | Local Step: 700 // 12101, total_eta: 87 days, 16:25:37, epoch_eta:19:50:18, time: all:5.999, data:0.001, vae:0.018, IQA:4.231, Connector:0.070, model:1.276, lr:3.041e-08, loss:0.559loss:0.3376, grad_norm:1.0537 | |
| 2025-10-24 02:05:14 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 800 | Local Step: 800 // 12101, total_eta: 87 days, 7:29:49, epoch_eta:19:34:57, time: all:6.056, data:0.001, vae:0.018, IQA:4.334, Connector:0.074, model:1.215, lr:3.536e-08, loss:0.395loss:0.3432, grad_norm:0.5366 | |
| 2025-10-24 02:15:18 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 900 | Local Step: 900 // 12101, total_eta: 87 days, 0:03:21, epoch_eta:19:20:31, time: all:6.043, data:0.001, vae:0.019, IQA:4.324, Connector:0.066, model:1.241, lr:3.960e-08, loss:0.387loss:0.3253, grad_norm:0.7253 | |
| 2025-10-24 02:25:42 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1000 | Local Step: 1000 // 12101, total_eta: 87 days, 0:43:44, epoch_eta:19:10:37, time: all:6.242, data:0.001, vae:0.018, IQA:4.302, Connector:0.079, model:1.345, lr:4.384e-08, loss:0.057loss:0.3265, grad_norm:0.5420 | |
| 2025-10-24 02:35:53 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1100 | Local Step: 1100 // 12101, total_eta: 86 days, 20:59:53, epoch_eta:18:58:19, time: all:6.102, data:0.001, vae:0.018, IQA:4.328, Connector:0.069, model:1.282, lr:4.808e-08, loss:0.015loss:0.3307, grad_norm:0.4225 | |
| 2025-10-24 02:45:53 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1200 | Local Step: 1200 // 12101, total_eta: 86 days, 15:12:10, epoch_eta:18:44:55, time: all:6.008, data:0.001, vae:0.018, IQA:4.300, Connector:0.069, model:1.222, lr:5.303e-08, loss:0.301loss:0.3274, grad_norm:0.9259 | |
| 2025-10-24 02:56:00 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1300 | Local Step: 1300 // 12101, total_eta: 86 days, 11:49:36, epoch_eta:18:32:53, time: all:6.068, data:0.001, vae:0.018, IQA:4.298, Connector:0.070, model:1.279, lr:5.728e-08, loss:0.174loss:0.3205, grad_norm:0.5335 | |
| 2025-10-24 03:06:05 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1400 | Local Step: 1400 // 12101, total_eta: 86 days, 8:30:14, epoch_eta:18:20:54, time: all:6.051, data:0.001, vae:0.018, IQA:4.220, Connector:0.073, model:1.322, lr:6.152e-08, loss:0.289loss:0.3101, grad_norm:0.4681 | |
| 2025-10-24 03:16:11 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1500 | Local Step: 1500 // 12101, total_eta: 86 days, 5:47:32, epoch_eta:18:09:17, time: all:6.059, data:0.001, vae:0.018, IQA:4.332, Connector:0.067, model:1.244, lr:6.576e-08, loss:0.254loss:0.3304, grad_norm:0.5355 | |
| 2025-10-24 03:26:20 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1600 | Local Step: 1600 // 12101, total_eta: 86 days, 4:01:33, epoch_eta:17:58:10, time: all:6.089, data:0.001, vae:0.018, IQA:4.255, Connector:0.068, model:1.337, lr:7.071e-08, loss:0.090loss:0.3200, grad_norm:0.7352 | |
| 2025-10-24 03:36:28 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1700 | Local Step: 1700 // 12101, total_eta: 86 days, 2:10:39, epoch_eta:17:47:02, time: all:6.076, data:0.001, vae:0.018, IQA:4.350, Connector:0.068, model:1.240, lr:7.495e-08, loss:0.426loss:0.3284, grad_norm:0.5895 | |
| 2025-10-24 03:46:33 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1800 | Local Step: 1800 // 12101, total_eta: 86 days, 0:05:33, epoch_eta:17:35:48, time: all:6.053, data:0.001, vae:0.018, IQA:4.345, Connector:0.068, model:1.217, lr:7.920e-08, loss:0.482loss:0.3201, grad_norm:0.6764 | |
| 2025-10-24 03:56:35 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 1900 | Local Step: 1900 // 12101, total_eta: 85 days, 21:42:42, epoch_eta:17:24:26, time: all:6.025, data:0.001, vae:0.018, IQA:4.253, Connector:0.069, model:1.271, lr:8.344e-08, loss:0.096loss:0.3292, grad_norm:1.4661 | |
| 2025-10-24 04:06:55 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2000 | Local Step: 2000 // 12101, total_eta: 85 days, 22:29:12, epoch_eta:17:14:40, time: all:6.200, data:0.001, vae:0.019, IQA:4.333, Connector:0.076, model:1.297, lr:8.839e-08, loss:0.324loss:0.3396, grad_norm:0.5887 | |
| 2025-10-24 04:17:08 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2100 | Local Step: 2100 // 12101, total_eta: 85 days, 22:03:35, epoch_eta:17:04:18, time: all:6.130, data:0.001, vae:0.018, IQA:4.384, Connector:0.069, model:1.264, lr:9.263e-08, loss:0.312loss:0.3216, grad_norm:0.3986 | |
| 2025-10-24 04:27:15 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2200 | Local Step: 2200 // 12101, total_eta: 85 days, 20:37:22, epoch_eta:16:53:26, time: all:6.062, data:0.001, vae:0.018, IQA:4.281, Connector:0.070, model:1.279, lr:9.687e-08, loss:0.193loss:0.3186, grad_norm:0.5003 | |
| 2025-10-24 04:37:19 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2300 | Local Step: 2300 // 12101, total_eta: 85 days, 19:05:43, epoch_eta:16:42:32, time: all:6.048, data:0.001, vae:0.018, IQA:4.354, Connector:0.068, model:1.222, lr:1.011e-07, loss:0.438loss:0.3333, grad_norm:0.4077 | |
| 2025-10-24 04:47:26 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2400 | Local Step: 2400 // 12101, total_eta: 85 days, 17:52:59, epoch_eta:16:31:48, time: all:6.063, data:0.001, vae:0.018, IQA:4.223, Connector:0.070, model:1.337, lr:1.061e-07, loss:0.363loss:0.3311, grad_norm:0.6398 | |
| 2025-10-24 04:57:30 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2500 | Local Step: 2500 // 12101, total_eta: 85 days, 16:30:44, epoch_eta:16:21:00, time: all:6.045, data:0.001, vae:0.018, IQA:4.252, Connector:0.066, model:1.301, lr:1.103e-07, loss:0.071loss:0.3238, grad_norm:0.5316 | |
| 2025-10-24 05:07:33 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2600 | Local Step: 2600 // 12101, total_eta: 85 days, 15:02:38, epoch_eta:16:10:11, time: all:6.030, data:0.001, vae:0.018, IQA:4.261, Connector:0.071, model:1.276, lr:1.146e-07, loss:0.182loss:0.3059, grad_norm:0.5973 | |
| 2025-10-24 05:17:38 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2700 | Local Step: 2700 // 12101, total_eta: 85 days, 13:56:37, epoch_eta:15:59:32, time: all:6.052, data:0.001, vae:0.018, IQA:4.292, Connector:0.071, model:1.271, lr:1.188e-07, loss:0.516loss:0.3431, grad_norm:0.4307 | |
| 2025-10-24 05:27:43 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2800 | Local Step: 2800 // 12101, total_eta: 85 days, 12:48:10, epoch_eta:15:48:53, time: all:6.043, data:0.001, vae:0.018, IQA:4.262, Connector:0.069, model:1.289, lr:1.237e-07, loss:0.175loss:0.3055, grad_norm:0.5092 | |
| 2025-10-24 05:37:48 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 2900 | Local Step: 2900 // 12101, total_eta: 85 days, 11:50:29, epoch_eta:15:38:19, time: all:6.053, data:0.001, vae:0.018, IQA:4.319, Connector:0.069, model:1.241, lr:1.280e-07, loss:0.084loss:0.3202, grad_norm:0.7259 | |
| 2025-10-24 05:48:11 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3000 | Local Step: 3000 // 12101, total_eta: 85 days, 12:53:37, epoch_eta:15:28:40, time: all:6.228, data:0.001, vae:0.018, IQA:4.347, Connector:0.076, model:1.303, lr:1.322e-07, loss:0.176loss:0.3252, grad_norm:0.6852 | |
| 2025-10-24 05:58:21 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3100 | Local Step: 3100 // 12101, total_eta: 85 days, 12:30:02, epoch_eta:15:18:22, time: all:6.102, data:0.001, vae:0.018, IQA:4.399, Connector:0.067, model:1.217, lr:1.365e-07, loss:0.285loss:0.3129, grad_norm:0.8307 | |
| 2025-10-24 06:08:27 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3200 | Local Step: 3200 // 12101, total_eta: 85 days, 11:43:15, epoch_eta:15:07:53, time: all:6.064, data:0.001, vae:0.018, IQA:4.364, Connector:0.067, model:1.214, lr:1.414e-07, loss:0.248loss:0.3109, grad_norm:0.4434 | |
| 2025-10-24 06:18:36 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3300 | Local Step: 3300 // 12101, total_eta: 85 days, 11:09:53, epoch_eta:14:57:31, time: all:6.082, data:0.001, vae:0.018, IQA:4.316, Connector:0.069, model:1.277, lr:1.457e-07, loss:0.305loss:0.3095, grad_norm:0.4146 | |
| 2025-10-24 06:28:40 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3400 | Local Step: 3400 // 12101, total_eta: 85 days, 10:15:23, epoch_eta:14:47:00, time: all:6.044, data:0.001, vae:0.018, IQA:4.345, Connector:0.067, model:1.218, lr:1.499e-07, loss:0.240loss:0.3349, grad_norm:0.4395 | |
| 2025-10-24 06:38:44 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3500 | Local Step: 3500 // 12101, total_eta: 85 days, 9:21:17, epoch_eta:14:36:30, time: all:6.040, data:0.001, vae:0.018, IQA:4.253, Connector:0.070, model:1.287, lr:1.541e-07, loss:0.125loss:0.3009, grad_norm:0.7243 | |
| 2025-10-24 06:48:48 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3600 | Local Step: 3600 // 12101, total_eta: 85 days, 8:28:27, epoch_eta:14:26:00, time: all:6.038, data:0.001, vae:0.018, IQA:4.268, Connector:0.071, model:1.265, lr:1.591e-07, loss:0.338loss:0.3293, grad_norm:0.5103 | |
| 2025-10-24 06:58:47 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3700 | Local Step: 3700 // 12101, total_eta: 85 days, 7:12:20, epoch_eta:14:15:22, time: all:5.991, data:0.001, vae:0.018, IQA:4.242, Connector:0.070, model:1.269, lr:1.633e-07, loss:0.283loss:0.3289, grad_norm:0.4714 | |
| 2025-10-24 07:08:51 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3800 | Local Step: 3800 // 12101, total_eta: 85 days, 6:27:43, epoch_eta:14:04:56, time: all:6.044, data:0.001, vae:0.018, IQA:4.318, Connector:0.069, model:1.235, lr:1.676e-07, loss:0.365loss:0.3231, grad_norm:0.7253 | |
| 2025-10-24 07:18:56 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 3900 | Local Step: 3900 // 12101, total_eta: 85 days, 5:44:16, epoch_eta:13:54:32, time: all:6.043, data:0.001, vae:0.018, IQA:4.327, Connector:0.067, model:1.243, lr:1.718e-07, loss:0.114loss:0.3139, grad_norm:0.4201 | |
| 2025-10-24 07:29:16 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4000 | Local Step: 4000 // 12101, total_eta: 85 days, 6:25:44, epoch_eta:13:44:42, time: all:6.208, data:0.001, vae:0.019, IQA:4.359, Connector:0.075, model:1.288, lr:1.768e-07, loss:0.773loss:0.3242, grad_norm:0.5798 | |
| 2025-10-24 07:39:19 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4100 | Local Step: 4100 // 12101, total_eta: 85 days, 5:35:19, epoch_eta:13:34:16, time: all:6.026, data:0.001, vae:0.018, IQA:4.249, Connector:0.069, model:1.284, lr:1.810e-07, loss:0.508loss:0.3217, grad_norm:0.6445 | |
| 2025-10-24 07:49:28 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4200 | Local Step: 4200 // 12101, total_eta: 85 days, 5:15:17, epoch_eta:13:24:01, time: all:6.086, data:0.001, vae:0.018, IQA:4.368, Connector:0.071, model:1.230, lr:1.853e-07, loss:0.414loss:0.3173, grad_norm:0.5173 | |
| 2025-10-24 07:59:27 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4300 | Local Step: 4300 // 12101, total_eta: 85 days, 4:12:22, epoch_eta:13:13:30, time: all:5.993, data:0.001, vae:0.018, IQA:4.305, Connector:0.067, model:1.210, lr:1.895e-07, loss:0.229loss:0.3248, grad_norm:1.3203 | |
| 2025-10-24 08:09:32 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4400 | Local Step: 4400 // 12101, total_eta: 85 days, 3:37:39, epoch_eta:13:03:10, time: all:6.049, data:0.001, vae:0.018, IQA:4.295, Connector:0.069, model:1.274, lr:1.945e-07, loss:0.633loss:0.3196, grad_norm:0.8136 | |
| 2025-10-24 08:19:41 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4500 | Local Step: 4500 // 12101, total_eta: 85 days, 3:23:53, epoch_eta:12:52:59, time: all:6.094, data:0.001, vae:0.018, IQA:4.313, Connector:0.069, model:1.285, lr:1.987e-07, loss:0.098loss:0.3439, grad_norm:0.6156 | |
| 2025-10-24 08:29:48 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4600 | Local Step: 4600 // 12101, total_eta: 85 days, 2:58:00, epoch_eta:12:42:43, time: all:6.066, data:0.001, vae:0.018, IQA:4.365, Connector:0.066, model:1.223, lr:2.029e-07, loss:0.122loss:0.3471, grad_norm:0.9479 | |
| 2025-10-24 08:39:50 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4700 | Local Step: 4700 // 12101, total_eta: 85 days, 2:14:39, epoch_eta:12:32:20, time: all:6.023, data:0.001, vae:0.018, IQA:4.283, Connector:0.069, model:1.252, lr:2.072e-07, loss:0.297loss:0.3241, grad_norm:0.5186 | |
| 2025-10-24 08:49:54 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4800 | Local Step: 4800 // 12101, total_eta: 85 days, 1:38:35, epoch_eta:12:22:01, time: all:6.037, data:0.001, vae:0.018, IQA:4.226, Connector:0.069, model:1.310, lr:2.121e-07, loss:1.281loss:0.3110, grad_norm:0.5865 | |
| 2025-10-24 08:59:59 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 4900 | Local Step: 4900 // 12101, total_eta: 85 days, 1:08:39, epoch_eta:12:11:44, time: all:6.050, data:0.001, vae:0.018, IQA:4.205, Connector:0.070, model:1.344, lr:2.164e-07, loss:0.672loss:0.3226, grad_norm:1.0345 | |
| 2025-10-24 09:10:06 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 1 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_1_step_5000_weight.pth. | |
| 2025-10-24 09:10:36 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 1 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_1_step_5000.pth. | |
| 2025-10-24 09:10:59 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5000 | Local Step: 5000 // 12101, total_eta: 85 days, 4:20:32, epoch_eta:12:02:46, time: all:6.600, data:0.001, vae:0.018, IQA:4.320, Connector:0.076, model:1.368, lr:2.206e-07, loss:0.355loss:0.3281, grad_norm:0.5999 | |
| 2025-10-24 09:21:06 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5100 | Local Step: 5100 // 12101, total_eta: 85 days, 3:55:43, epoch_eta:11:52:30, time: all:6.070, data:0.001, vae:0.018, IQA:4.345, Connector:0.066, model:1.228, lr:2.249e-07, loss:0.178loss:0.3145, grad_norm:0.5026 | |
| 2025-10-24 09:31:18 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5200 | Local Step: 5200 // 12101, total_eta: 85 days, 3:51:02, epoch_eta:11:42:21, time: all:6.121, data:0.001, vae:0.018, IQA:4.371, Connector:0.067, model:1.273, lr:2.298e-07, loss:0.017loss:0.3237, grad_norm:0.4886 | |
| 2025-10-24 09:41:25 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5300 | Local Step: 5300 // 12101, total_eta: 85 days, 3:29:07, epoch_eta:11:32:07, time: all:6.076, data:0.001, vae:0.018, IQA:4.318, Connector:0.069, model:1.269, lr:2.341e-07, loss:0.139loss:0.3268, grad_norm:0.4902 | |
| 2025-10-24 09:51:34 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5400 | Local Step: 5400 // 12101, total_eta: 85 days, 3:11:21, epoch_eta:11:21:54, time: all:6.086, data:0.001, vae:0.018, IQA:4.287, Connector:0.067, model:1.314, lr:2.383e-07, loss:0.199loss:0.3057, grad_norm:0.4629 | |
| 2025-10-24 10:01:38 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5500 | Local Step: 5500 // 12101, total_eta: 85 days, 2:37:16, epoch_eta:11:11:35, time: all:6.040, data:0.001, vae:0.017, IQA:4.279, Connector:0.066, model:1.271, lr:2.425e-07, loss:0.469loss:0.3171, grad_norm:0.6094 | |
| 2025-10-24 10:11:44 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5600 | Local Step: 5600 // 12101, total_eta: 85 days, 2:09:37, epoch_eta:11:01:19, time: all:6.056, data:0.001, vae:0.019, IQA:4.286, Connector:0.068, model:1.280, lr:2.475e-07, loss:0.242loss:0.3344, grad_norm:0.5254 | |
| 2025-10-24 10:21:54 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5700 | Local Step: 5700 // 12101, total_eta: 85 days, 1:59:26, epoch_eta:10:51:09, time: all:6.104, data:0.001, vae:0.018, IQA:4.322, Connector:0.068, model:1.300, lr:2.517e-07, loss:0.014loss:0.3240, grad_norm:0.6712 | |
| 2025-10-24 10:32:02 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5800 | Local Step: 5800 // 12101, total_eta: 85 days, 1:42:39, epoch_eta:10:40:56, time: all:6.084, data:0.001, vae:0.018, IQA:4.350, Connector:0.072, model:1.240, lr:2.560e-07, loss:0.207loss:0.3154, grad_norm:0.5316 | |
| 2025-10-24 10:42:08 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 5900 | Local Step: 5900 // 12101, total_eta: 85 days, 1:17:34, epoch_eta:10:30:41, time: all:6.059, data:0.001, vae:0.017, IQA:4.259, Connector:0.068, model:1.302, lr:2.602e-07, loss:0.256loss:0.3060, grad_norm:2.7637 | |
| 2025-10-24 10:52:24 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6000 | Local Step: 6000 // 12101, total_eta: 85 days, 1:25:51, epoch_eta:10:20:37, time: all:6.158, data:0.001, vae:0.019, IQA:4.260, Connector:0.079, model:1.331, lr:2.652e-07, loss:0.091loss:0.3211, grad_norm:0.6013 | |
| 2025-10-24 11:02:34 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6100 | Local Step: 6100 // 12101, total_eta: 85 days, 1:15:23, epoch_eta:10:10:26, time: all:6.103, data:0.001, vae:0.018, IQA:4.346, Connector:0.069, model:1.279, lr:2.694e-07, loss:0.289loss:0.3253, grad_norm:1.4762 | |
| 2025-10-24 11:12:38 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6200 | Local Step: 6200 // 12101, total_eta: 85 days, 0:43:07, epoch_eta:10:00:09, time: all:6.035, data:0.001, vae:0.018, IQA:4.372, Connector:0.069, model:1.193, lr:2.737e-07, loss:0.191loss:0.3197, grad_norm:0.3132 | |
| 2025-10-24 11:22:39 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6300 | Local Step: 6300 // 12101, total_eta: 85 days, 0:03:55, epoch_eta:9:49:51, time: all:6.011, data:0.001, vae:0.018, IQA:4.359, Connector:0.068, model:1.188, lr:2.779e-07, loss:0.019loss:0.3167, grad_norm:0.6702 | |
| 2025-10-24 11:32:39 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6400 | Local Step: 6400 // 12101, total_eta: 84 days, 23:23:37, epoch_eta:9:39:32, time: all:6.005, data:0.001, vae:0.018, IQA:4.270, Connector:0.071, model:1.272, lr:2.828e-07, loss:0.344loss:0.3117, grad_norm:0.6588 | |
| 2025-10-24 11:42:44 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6500 | Local Step: 6500 // 12101, total_eta: 84 days, 22:57:06, epoch_eta:9:29:18, time: all:6.046, data:0.001, vae:0.018, IQA:4.294, Connector:0.068, model:1.278, lr:2.871e-07, loss:0.098loss:0.3000, grad_norm:0.3867 | |
| 2025-10-24 11:52:48 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6600 | Local Step: 6600 // 12101, total_eta: 84 days, 22:29:58, epoch_eta:9:19:03, time: all:6.043, data:0.001, vae:0.018, IQA:4.398, Connector:0.068, model:1.182, lr:2.913e-07, loss:0.559loss:0.3129, grad_norm:0.4144 | |
| 2025-10-24 12:02:49 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6700 | Local Step: 6700 // 12101, total_eta: 84 days, 21:53:23, epoch_eta:9:08:46, time: all:6.009, data:0.001, vae:0.018, IQA:4.345, Connector:0.067, model:1.204, lr:2.956e-07, loss:0.184loss:0.3194, grad_norm:0.8421 | |
| 2025-10-24 12:12:54 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6800 | Local Step: 6800 // 12101, total_eta: 84 days, 21:28:58, epoch_eta:8:58:33, time: all:6.048, data:0.001, vae:0.018, IQA:4.343, Connector:0.068, model:1.234, lr:3.005e-07, loss:0.457loss:0.3061, grad_norm:0.5508 | |
| 2025-10-24 12:22:58 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 6900 | Local Step: 6900 // 12101, total_eta: 84 days, 21:02:51, epoch_eta:8:48:19, time: all:6.041, data:0.001, vae:0.019, IQA:4.317, Connector:0.070, model:1.256, lr:3.048e-07, loss:0.426loss:0.3300, grad_norm:1.3831 | |
| 2025-10-24 12:33:20 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7000 | Local Step: 7000 // 12101, total_eta: 84 days, 21:27:48, epoch_eta:8:38:18, time: all:6.217, data:0.001, vae:0.019, IQA:4.392, Connector:0.074, model:1.281, lr:3.090e-07, loss:0.379loss:0.3215, grad_norm:1.3452 | |
| 2025-10-24 12:43:24 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7100 | Local Step: 7100 // 12101, total_eta: 84 days, 21:01:05, epoch_eta:8:28:05, time: all:6.038, data:0.001, vae:0.019, IQA:4.317, Connector:0.067, model:1.249, lr:3.132e-07, loss:0.141loss:0.3096, grad_norm:0.5036 | |
| 2025-10-24 12:53:27 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7200 | Local Step: 7200 // 12101, total_eta: 84 days, 20:32:35, epoch_eta:8:17:51, time: all:6.030, data:0.001, vae:0.018, IQA:4.265, Connector:0.071, model:1.284, lr:3.182e-07, loss:0.469loss:0.3077, grad_norm:0.5145 | |
| 2025-10-24 13:03:32 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7300 | Local Step: 7300 // 12101, total_eta: 84 days, 20:10:05, epoch_eta:8:07:38, time: all:6.050, data:0.001, vae:0.018, IQA:4.345, Connector:0.068, model:1.233, lr:3.224e-07, loss:0.077loss:0.3122, grad_norm:1.2786 | |
| 2025-10-24 13:13:32 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7400 | Local Step: 7400 // 12101, total_eta: 84 days, 19:36:10, epoch_eta:7:57:23, time: all:6.007, data:0.001, vae:0.018, IQA:4.170, Connector:0.067, model:1.362, lr:3.267e-07, loss:0.142loss:0.3259, grad_norm:0.3846 | |
| 2025-10-24 13:23:40 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7500 | Local Step: 7500 // 12101, total_eta: 84 days, 19:21:41, epoch_eta:7:47:13, time: all:6.077, data:0.001, vae:0.018, IQA:4.372, Connector:0.067, model:1.241, lr:3.309e-07, loss:0.176loss:0.3197, grad_norm:0.6182 | |
| 2025-10-24 13:33:46 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7600 | Local Step: 7600 // 12101, total_eta: 84 days, 19:03:22, epoch_eta:7:37:02, time: all:6.062, data:0.001, vae:0.018, IQA:4.348, Connector:0.069, model:1.251, lr:3.359e-07, loss:0.254loss:0.3170, grad_norm:0.3557 | |
| 2025-10-24 13:43:52 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7700 | Local Step: 7700 // 12101, total_eta: 84 days, 18:45:10, epoch_eta:7:26:51, time: all:6.062, data:0.001, vae:0.018, IQA:4.375, Connector:0.068, model:1.232, lr:3.401e-07, loss:0.113loss:0.3043, grad_norm:0.5999 | |
| 2025-10-24 13:53:56 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7800 | Local Step: 7800 // 12101, total_eta: 84 days, 18:19:29, epoch_eta:7:16:38, time: all:6.032, data:0.001, vae:0.019, IQA:4.297, Connector:0.070, model:1.258, lr:3.444e-07, loss:0.225loss:0.3315, grad_norm:0.4375 | |
| 2025-10-24 14:03:55 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 7900 | Local Step: 7900 // 12101, total_eta: 84 days, 17:43:22, epoch_eta:7:06:24, time: all:5.989, data:0.001, vae:0.018, IQA:4.301, Connector:0.066, model:1.231, lr:3.486e-07, loss:0.017loss:0.3136, grad_norm:0.6844 | |
| 2025-10-24 14:14:07 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8000 | Local Step: 8000 // 12101, total_eta: 84 days, 17:41:25, epoch_eta:6:56:16, time: all:6.123, data:0.001, vae:0.019, IQA:4.224, Connector:0.076, model:1.340, lr:3.536e-07, loss:0.158loss:0.3237, grad_norm:0.5301 | |
| 2025-10-24 14:24:15 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8100 | Local Step: 8100 // 12101, total_eta: 84 days, 17:28:57, epoch_eta:6:46:07, time: all:6.081, data:0.001, vae:0.018, IQA:4.286, Connector:0.070, model:1.308, lr:3.578e-07, loss:0.453loss:0.3181, grad_norm:0.4863 | |
| 2025-10-24 14:34:22 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8200 | Local Step: 8200 // 12101, total_eta: 84 days, 17:14:55, epoch_eta:6:35:57, time: all:6.074, data:0.001, vae:0.018, IQA:4.347, Connector:0.068, model:1.255, lr:3.620e-07, loss:0.201loss:0.3109, grad_norm:0.7568 | |
| 2025-10-24 14:44:26 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8300 | Local Step: 8300 // 12101, total_eta: 84 days, 16:50:51, epoch_eta:6:25:45, time: all:6.032, data:0.001, vae:0.018, IQA:4.285, Connector:0.070, model:1.268, lr:3.663e-07, loss:0.126loss:0.3152, grad_norm:0.4219 | |
| 2025-10-24 14:54:28 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8400 | Local Step: 8400 // 12101, total_eta: 84 days, 16:26:03, epoch_eta:6:15:34, time: all:6.028, data:0.001, vae:0.018, IQA:4.365, Connector:0.069, model:1.192, lr:3.712e-07, loss:0.248loss:0.3124, grad_norm:0.4428 | |
| 2025-10-24 15:04:29 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8500 | Local Step: 8500 // 12101, total_eta: 84 days, 15:55:29, epoch_eta:6:05:21, time: all:6.002, data:0.001, vae:0.018, IQA:4.262, Connector:0.072, model:1.269, lr:3.755e-07, loss:0.240loss:0.3156, grad_norm:0.6126 | |
| 2025-10-24 15:14:31 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8600 | Local Step: 8600 // 12101, total_eta: 84 days, 15:30:19, epoch_eta:5:55:10, time: all:6.023, data:0.001, vae:0.018, IQA:4.319, Connector:0.068, model:1.236, lr:3.797e-07, loss:0.350loss:0.3211, grad_norm:0.2979 | |
| 2025-10-24 15:24:33 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8700 | Local Step: 8700 // 12101, total_eta: 84 days, 15:05:14, epoch_eta:5:44:59, time: all:6.022, data:0.001, vae:0.018, IQA:4.209, Connector:0.070, model:1.334, lr:3.840e-07, loss:0.688loss:0.3181, grad_norm:0.8174 | |
| 2025-10-24 15:34:34 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8800 | Local Step: 8800 // 12101, total_eta: 84 days, 14:37:44, epoch_eta:5:34:47, time: all:6.010, data:0.001, vae:0.018, IQA:4.332, Connector:0.070, model:1.214, lr:3.889e-07, loss:0.146loss:0.3229, grad_norm:0.5172 | |
| 2025-10-24 15:44:33 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 8900 | Local Step: 8900 // 12101, total_eta: 84 days, 14:05:56, epoch_eta:5:24:35, time: all:5.989, data:0.001, vae:0.018, IQA:4.228, Connector:0.070, model:1.285, lr:3.932e-07, loss:0.342loss:0.3132, grad_norm:0.5312 | |
| 2025-10-24 15:54:55 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9000 | Local Step: 9000 // 12101, total_eta: 84 days, 14:25:39, epoch_eta:5:14:31, time: all:6.218, data:0.001, vae:0.020, IQA:4.378, Connector:0.075, model:1.298, lr:3.974e-07, loss:0.293loss:0.3238, grad_norm:0.8701 | |
| 2025-10-24 16:04:56 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9100 | Local Step: 9100 // 12101, total_eta: 84 days, 13:58:20, epoch_eta:5:04:20, time: all:6.008, data:0.001, vae:0.019, IQA:4.343, Connector:0.070, model:1.200, lr:4.016e-07, loss:0.266loss:0.3193, grad_norm:0.8148 | |
| 2025-10-24 16:15:00 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9200 | Local Step: 9200 // 12101, total_eta: 84 days, 13:39:24, epoch_eta:4:54:10, time: all:6.044, data:0.001, vae:0.018, IQA:4.356, Connector:0.068, model:1.211, lr:4.066e-07, loss:0.586loss:0.3039, grad_norm:0.8418 | |
| 2025-10-24 16:25:02 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9300 | Local Step: 9300 // 12101, total_eta: 84 days, 13:16:13, epoch_eta:4:44:00, time: all:6.024, data:0.001, vae:0.019, IQA:4.297, Connector:0.070, model:1.254, lr:4.108e-07, loss:0.066loss:0.3372, grad_norm:0.6279 | |
| 2025-10-24 16:35:06 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9400 | Local Step: 9400 // 12101, total_eta: 84 days, 12:56:16, epoch_eta:4:33:50, time: all:6.038, data:0.001, vae:0.018, IQA:4.303, Connector:0.067, model:1.264, lr:4.151e-07, loss:0.106loss:0.3013, grad_norm:0.8688 | |
| 2025-10-24 16:45:05 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9500 | Local Step: 9500 // 12101, total_eta: 84 days, 12:26:42, epoch_eta:4:23:40, time: all:5.991, data:0.001, vae:0.018, IQA:4.280, Connector:0.068, model:1.236, lr:4.193e-07, loss:0.114loss:0.3141, grad_norm:1.0931 | |
| 2025-10-24 16:55:08 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9600 | Local Step: 9600 // 12101, total_eta: 84 days, 12:04:18, epoch_eta:4:13:30, time: all:6.023, data:0.001, vae:0.019, IQA:4.317, Connector:0.070, model:1.225, lr:4.243e-07, loss:0.270loss:0.3160, grad_norm:0.6205 | |
| 2025-10-24 17:05:13 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9700 | Local Step: 9700 // 12101, total_eta: 84 days, 11:48:21, epoch_eta:4:03:21, time: all:6.053, data:0.001, vae:0.018, IQA:4.366, Connector:0.067, model:1.224, lr:4.285e-07, loss:0.168loss:0.3330, grad_norm:0.4473 | |
| 2025-10-24 17:15:19 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9800 | Local Step: 9800 // 12101, total_eta: 84 days, 11:32:54, epoch_eta:3:53:12, time: all:6.055, data:0.001, vae:0.018, IQA:4.389, Connector:0.068, model:1.202, lr:4.327e-07, loss:0.143loss:0.3115, grad_norm:0.4635 | |
| 2025-10-24 17:25:18 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 9900 | Local Step: 9900 // 12101, total_eta: 84 days, 11:06:00, epoch_eta:3:43:02, time: all:5.998, data:0.001, vae:0.018, IQA:4.300, Connector:0.068, model:1.232, lr:4.370e-07, loss:0.236loss:0.3192, grad_norm:0.4800 | |
| 2025-10-24 17:35:19 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 1 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_1_step_10000_weight.pth. | |
| 2025-10-24 17:35:47 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 1 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_1_step_10000.pth. | |
| 2025-10-24 17:36:07 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10000 | Local Step: 10000 // 12101, total_eta: 84 days, 12:17:15, epoch_eta:3:33:03, time: all:6.487, data:0.001, vae:0.019, IQA:4.340, Connector:0.074, model:1.265, lr:4.419e-07, loss:1.469loss:0.3217, grad_norm:0.5578 | |
| 2025-10-24 17:46:13 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10100 | Local Step: 10100 // 12101, total_eta: 84 days, 12:01:49, epoch_eta:3:22:54, time: all:6.058, data:0.001, vae:0.019, IQA:4.318, Connector:0.070, model:1.266, lr:4.462e-07, loss:0.107loss:0.3168, grad_norm:0.4915 | |
| 2025-10-24 17:56:17 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10200 | Local Step: 10200 // 12101, total_eta: 84 days, 11:44:01, epoch_eta:3:12:45, time: all:6.045, data:0.001, vae:0.019, IQA:4.216, Connector:0.070, model:1.358, lr:4.504e-07, loss:0.875loss:0.3163, grad_norm:0.5140 | |
| 2025-10-24 18:06:21 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10300 | Local Step: 10300 // 12101, total_eta: 84 days, 11:24:22, epoch_eta:3:02:35, time: all:6.035, data:0.001, vae:0.019, IQA:4.320, Connector:0.070, model:1.238, lr:4.547e-07, loss:0.154loss:0.2918, grad_norm:0.4163 | |
| 2025-10-24 18:16:29 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10400 | Local Step: 10400 // 12101, total_eta: 84 days, 11:13:07, epoch_eta:2:52:27, time: all:6.077, data:0.001, vae:0.018, IQA:4.375, Connector:0.072, model:1.233, lr:4.596e-07, loss:1.117loss:0.2996, grad_norm:0.7335 | |
| 2025-10-24 18:26:33 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10500 | Local Step: 10500 // 12101, total_eta: 84 days, 10:56:12, epoch_eta:2:42:18, time: all:6.048, data:0.001, vae:0.018, IQA:4.297, Connector:0.067, model:1.279, lr:4.639e-07, loss:0.209loss:0.3236, grad_norm:0.5977 | |
| 2025-10-24 18:36:36 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10600 | Local Step: 10600 // 12101, total_eta: 84 days, 10:36:09, epoch_eta:2:32:09, time: all:6.030, data:0.001, vae:0.018, IQA:4.252, Connector:0.068, model:1.310, lr:4.681e-07, loss:0.102loss:0.3244, grad_norm:1.3473 | |
| 2025-10-24 18:46:35 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10700 | Local Step: 10700 // 12101, total_eta: 84 days, 10:07:58, epoch_eta:2:22:00, time: all:5.986, data:0.001, vae:0.019, IQA:4.253, Connector:0.068, model:1.256, lr:4.723e-07, loss:0.609loss:0.3232, grad_norm:0.9157 | |
| 2025-10-24 18:56:42 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10800 | Local Step: 10800 // 12101, total_eta: 84 days, 9:56:04, epoch_eta:2:11:51, time: all:6.072, data:0.001, vae:0.018, IQA:4.368, Connector:0.071, model:1.238, lr:4.773e-07, loss:0.007loss:0.3119, grad_norm:0.4637 | |
| 2025-10-24 19:06:50 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 10900 | Local Step: 10900 // 12101, total_eta: 84 days, 9:44:46, epoch_eta:2:01:43, time: all:6.075, data:0.001, vae:0.019, IQA:4.327, Connector:0.070, model:1.280, lr:4.815e-07, loss:1.188loss:0.3052, grad_norm:0.5505 | |
| 2025-10-24 19:17:10 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11000 | Local Step: 11000 // 12101, total_eta: 84 days, 9:57:20, epoch_eta:1:51:36, time: all:6.206, data:0.001, vae:0.019, IQA:4.278, Connector:0.075, model:1.380, lr:4.858e-07, loss:0.344loss:0.3158, grad_norm:1.0579 | |
| 2025-10-24 19:27:09 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11100 | Local Step: 11100 // 12101, total_eta: 84 days, 9:30:57, epoch_eta:1:41:27, time: all:5.992, data:0.001, vae:0.018, IQA:4.245, Connector:0.071, model:1.274, lr:4.900e-07, loss:0.244loss:0.3079, grad_norm:0.6217 | |
| 2025-10-24 19:37:13 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11200 | Local Step: 11200 // 12101, total_eta: 84 days, 9:12:59, epoch_eta:1:31:19, time: all:6.038, data:0.001, vae:0.018, IQA:4.340, Connector:0.066, model:1.239, lr:4.950e-07, loss:0.243loss:0.3196, grad_norm:0.4986 | |
| 2025-10-24 19:47:17 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11300 | Local Step: 11300 // 12101, total_eta: 84 days, 8:54:42, epoch_eta:1:21:10, time: all:6.035, data:0.001, vae:0.019, IQA:4.306, Connector:0.067, model:1.261, lr:4.992e-07, loss:0.197loss:0.3122, grad_norm:0.5226 | |
| 2025-10-24 19:57:19 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11400 | Local Step: 11400 // 12101, total_eta: 84 days, 8:33:37, epoch_eta:1:11:02, time: all:6.018, data:0.001, vae:0.018, IQA:4.226, Connector:0.070, model:1.298, lr:5.035e-07, loss:0.060loss:0.3073, grad_norm:0.3737 | |
| 2025-10-24 20:07:26 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11500 | Local Step: 11500 // 12101, total_eta: 84 days, 8:23:18, epoch_eta:1:00:54, time: all:6.079, data:0.001, vae:0.019, IQA:4.315, Connector:0.071, model:1.275, lr:5.077e-07, loss:0.250loss:0.3139, grad_norm:0.9209 | |
| 2025-10-24 20:17:33 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11600 | Local Step: 11600 // 12101, total_eta: 84 days, 8:09:47, epoch_eta:0:50:46, time: all:6.061, data:0.001, vae:0.018, IQA:4.348, Connector:0.067, model:1.237, lr:5.127e-07, loss:0.482loss:0.3115, grad_norm:0.6702 | |
| 2025-10-24 20:27:32 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11700 | Local Step: 11700 // 12101, total_eta: 84 days, 7:45:03, epoch_eta:0:40:37, time: all:5.995, data:0.001, vae:0.018, IQA:4.273, Connector:0.067, model:1.245, lr:5.169e-07, loss:0.031loss:0.3079, grad_norm:0.3916 | |
| 2025-10-24 20:37:32 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11800 | Local Step: 11800 // 12101, total_eta: 84 days, 7:21:27, epoch_eta:0:30:29, time: all:6.000, data:0.001, vae:0.018, IQA:4.284, Connector:0.071, model:1.234, lr:5.211e-07, loss:0.244loss:0.3221, grad_norm:0.5094 | |
| 2025-10-24 20:47:36 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 11900 | Local Step: 11900 // 12101, total_eta: 84 days, 7:04:49, epoch_eta:0:20:21, time: all:6.040, data:0.001, vae:0.018, IQA:4.299, Connector:0.070, model:1.261, lr:5.254e-07, loss:0.207loss:0.3104, grad_norm:0.5290 | |
| 2025-10-24 20:57:48 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 12000 | Local Step: 12000 // 12101, total_eta: 84 days, 7:02:19, epoch_eta:0:10:13, time: all:6.124, data:0.001, vae:0.019, IQA:4.246, Connector:0.074, model:1.328, lr:5.303e-07, loss:0.344loss:0.3106, grad_norm:0.4813 | |
| 2025-10-24 21:07:52 - [1m[Sana][0m - INFO - Epoch: 1 | Global Step: 12100 | Local Step: 12100 // 12101, total_eta: 84 days, 6:45:36, epoch_eta:0:00:06, time: all:6.039, data:0.001, vae:0.018, IQA:4.288, Connector:0.069, model:1.277, lr:5.346e-07, loss:0.207loss:0.3255, grad_norm:0.9183 | |
| 2025-10-24 21:08:05 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 1 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_1_step_12102_weight.pth. | |
| 2025-10-24 21:08:35 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 1 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_1_step_12102.pth. | |
| 2025-10-24 21:08:43 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12102 | Local Step: 1 // 12101, total_eta: 109 days, 3:41:24, epoch_eta:1 day, 2:27:33, time: all:0.079, data:0.015, vae:0.000, IQA:0.048, Connector:0.001, model:0.010, lr:5.346e-07, loss:0.879loss:0.2822 | |
| 2025-10-24 21:18:54 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12201 | Local Step: 100 // 12101, total_eta: 85 days, 18:25:46, epoch_eta:20:37:19, time: all:6.107, data:0.001, vae:0.020, IQA:4.255, Connector:0.073, model:1.312, lr:5.388e-07, loss:0.248loss:0.3171, grad_norm:0.5889 | |
| 2025-10-24 21:28:57 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12301 | Local Step: 200 // 12101, total_eta: 84 days, 17:07:02, epoch_eta:20:12:01, time: all:6.035, data:0.001, vae:0.018, IQA:4.289, Connector:0.068, model:1.278, lr:5.431e-07, loss:0.777loss:0.3052, grad_norm:27.6120 | |
| 2025-10-24 21:38:59 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12401 | Local Step: 300 // 12101, total_eta: 84 days, 6:36:45, epoch_eta:19:55:44, time: all:6.017, data:0.001, vae:0.019, IQA:4.296, Connector:0.070, model:1.244, lr:5.480e-07, loss:0.017loss:0.3158, grad_norm:1.1108 | |
| 2025-10-24 21:49:00 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12501 | Local Step: 400 // 12101, total_eta: 84 days, 1:04:36, epoch_eta:19:42:27, time: all:6.015, data:0.001, vae:0.018, IQA:4.279, Connector:0.069, model:1.264, lr:5.523e-07, loss:0.322loss:0.3142, grad_norm:0.3454 | |
| 2025-10-24 21:59:01 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12601 | Local Step: 500 // 12101, total_eta: 83 days, 21:06:42, epoch_eta:19:30:08, time: all:6.006, data:0.001, vae:0.018, IQA:4.298, Connector:0.071, model:1.242, lr:5.565e-07, loss:0.348loss:0.3268, grad_norm:0.3996 | |
| 2025-10-24 22:09:02 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12701 | Local Step: 600 // 12101, total_eta: 83 days, 18:21:23, epoch_eta:19:18:34, time: all:6.005, data:0.001, vae:0.018, IQA:4.266, Connector:0.068, model:1.274, lr:5.607e-07, loss:0.242loss:0.3146, grad_norm:0.4907 | |
| 2025-10-24 22:19:06 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12801 | Local Step: 700 // 12101, total_eta: 83 days, 18:20:18, epoch_eta:19:08:34, time: all:6.047, data:0.001, vae:0.018, IQA:4.171, Connector:0.070, model:1.372, lr:5.657e-07, loss:0.192loss:0.3199, grad_norm:0.4510 | |
| 2025-10-24 22:29:09 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 12901 | Local Step: 800 // 12101, total_eta: 83 days, 17:17:59, epoch_eta:18:58:00, time: all:6.024, data:0.001, vae:0.018, IQA:4.274, Connector:0.070, model:1.260, lr:5.699e-07, loss:0.130loss:0.2939, grad_norm:1.5264 | |
| 2025-10-24 22:39:51 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13001 | Local Step: 900 // 12101, total_eta: 84 days, 7:03:12, epoch_eta:18:55:45, time: all:6.419, data:0.001, vae:0.019, IQA:4.405, Connector:0.083, model:1.389, lr:5.742e-07, loss:0.264loss:0.3304, grad_norm:0.4611 | |
| 2025-10-24 22:49:57 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13101 | Local Step: 1000 // 12101, total_eta: 84 days, 6:23:10, epoch_eta:18:45:20, time: all:6.069, data:0.001, vae:0.018, IQA:4.313, Connector:0.069, model:1.264, lr:5.784e-07, loss:0.007loss:0.3169, grad_norm:0.8669 | |
| 2025-10-24 23:00:01 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13201 | Local Step: 1100 // 12101, total_eta: 84 days, 4:42:02, epoch_eta:18:34:22, time: all:6.032, data:0.001, vae:0.018, IQA:4.279, Connector:0.071, model:1.261, lr:5.834e-07, loss:0.017loss:0.3280, grad_norm:0.4455 | |
| 2025-10-24 23:10:06 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13301 | Local Step: 1200 // 12101, total_eta: 84 days, 3:56:07, epoch_eta:18:23:54, time: all:6.056, data:0.001, vae:0.019, IQA:4.329, Connector:0.069, model:1.233, lr:5.876e-07, loss:0.183loss:0.3026, grad_norm:0.5332 | |
| 2025-10-24 23:20:15 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13401 | Local Step: 1300 // 12101, total_eta: 84 days, 4:04:08, epoch_eta:18:13:56, time: all:6.088, data:0.001, vae:0.018, IQA:4.354, Connector:0.068, model:1.248, lr:5.918e-07, loss:0.369loss:0.3304, grad_norm:0.7214 | |
| 2025-10-24 23:30:20 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13501 | Local Step: 1400 // 12101, total_eta: 84 days, 3:09:26, epoch_eta:18:03:25, time: all:6.046, data:0.001, vae:0.018, IQA:4.284, Connector:0.071, model:1.282, lr:5.961e-07, loss:0.395loss:0.2979, grad_norm:0.5400 | |
| 2025-10-24 23:40:24 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13601 | Local Step: 1500 // 12101, total_eta: 84 days, 2:14:18, epoch_eta:17:52:53, time: all:6.041, data:0.001, vae:0.018, IQA:4.274, Connector:0.070, model:1.270, lr:6.010e-07, loss:0.367loss:0.3107, grad_norm:0.4354 | |
| 2025-10-24 23:50:29 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13701 | Local Step: 1600 // 12101, total_eta: 84 days, 1:33:56, epoch_eta:17:42:30, time: all:6.048, data:0.001, vae:0.018, IQA:4.282, Connector:0.072, model:1.280, lr:6.053e-07, loss:0.022loss:0.3282, grad_norm:0.7161 | |
| 2025-10-25 00:00:36 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13801 | Local Step: 1700 // 12101, total_eta: 84 days, 1:26:38, epoch_eta:17:32:25, time: all:6.073, data:0.001, vae:0.018, IQA:4.332, Connector:0.068, model:1.254, lr:6.095e-07, loss:0.656loss:0.3292, grad_norm:0.8047 | |
| 2025-10-25 00:10:43 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 13901 | Local Step: 1800 // 12101, total_eta: 84 days, 1:14:04, epoch_eta:17:22:16, time: all:6.069, data:0.001, vae:0.018, IQA:4.317, Connector:0.068, model:1.271, lr:6.138e-07, loss:0.238loss:0.3313, grad_norm:0.8115 | |
| 2025-10-25 00:21:08 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14001 | Local Step: 1900 // 12101, total_eta: 84 days, 4:13:32, epoch_eta:17:13:46, time: all:6.252, data:0.001, vae:0.019, IQA:4.365, Connector:0.074, model:1.340, lr:6.187e-07, loss:0.336loss:0.3231, grad_norm:0.4827 | |
| 2025-10-25 00:31:18 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14101 | Local Step: 2000 // 12101, total_eta: 84 days, 4:20:47, epoch_eta:17:03:47, time: all:6.098, data:0.001, vae:0.018, IQA:4.342, Connector:0.070, model:1.291, lr:6.230e-07, loss:0.332loss:0.3089, grad_norm:1.9199 | |
| 2025-10-25 00:41:20 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14201 | Local Step: 2100 // 12101, total_eta: 84 days, 3:17:20, epoch_eta:16:53:12, time: all:6.025, data:0.001, vae:0.019, IQA:4.277, Connector:0.070, model:1.285, lr:6.272e-07, loss:0.189loss:0.3163, grad_norm:0.4769 | |
| 2025-10-25 00:51:24 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14301 | Local Step: 2200 // 12101, total_eta: 84 days, 2:25:34, epoch_eta:16:42:43, time: all:6.033, data:0.001, vae:0.018, IQA:4.329, Connector:0.069, model:1.246, lr:6.314e-07, loss:0.277loss:0.3195, grad_norm:0.4027 | |
| 2025-10-25 01:01:32 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14401 | Local Step: 2300 // 12101, total_eta: 84 days, 2:25:01, epoch_eta:16:32:41, time: all:6.088, data:0.001, vae:0.018, IQA:4.258, Connector:0.071, model:1.361, lr:6.364e-07, loss:0.336loss:0.3114, grad_norm:0.4503 | |
| 2025-10-25 01:11:37 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14501 | Local Step: 2400 // 12101, total_eta: 84 days, 1:54:05, epoch_eta:16:22:23, time: all:6.052, data:0.001, vae:0.018, IQA:4.340, Connector:0.068, model:1.248, lr:6.406e-07, loss:0.172loss:0.3092, grad_norm:0.4290 | |
| 2025-10-25 01:21:41 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14601 | Local Step: 2500 // 12101, total_eta: 84 days, 1:13:26, epoch_eta:16:12:00, time: all:6.038, data:0.001, vae:0.018, IQA:4.192, Connector:0.068, model:1.365, lr:6.449e-07, loss:0.316loss:0.3204, grad_norm:0.4131 | |
| 2025-10-25 01:31:46 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14701 | Local Step: 2600 // 12101, total_eta: 84 days, 0:41:03, epoch_eta:16:01:42, time: all:6.045, data:0.001, vae:0.018, IQA:4.349, Connector:0.068, model:1.242, lr:6.491e-07, loss:0.044loss:0.3032, grad_norm:0.5710 | |
| 2025-10-25 01:41:50 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14801 | Local Step: 2700 // 12101, total_eta: 84 days, 0:08:07, epoch_eta:15:51:24, time: all:6.042, data:0.001, vae:0.018, IQA:4.284, Connector:0.068, model:1.293, lr:6.541e-07, loss:0.168loss:0.3260, grad_norm:0.4459 | |
| 2025-10-25 01:51:55 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 14901 | Local Step: 2800 // 12101, total_eta: 83 days, 23:41:45, epoch_eta:15:41:09, time: all:6.049, data:0.001, vae:0.018, IQA:4.308, Connector:0.070, model:1.271, lr:6.583e-07, loss:0.187loss:0.3129, grad_norm:0.4902 | |
| 2025-10-25 02:01:57 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 2 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_2_step_15000_weight.pth. | |
| 2025-10-25 02:02:26 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 2 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_2_step_15000.pth. | |
| 2025-10-25 02:02:55 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15001 | Local Step: 2900 // 12101, total_eta: 84 days, 5:37:57, epoch_eta:15:33:52, time: all:6.605, data:0.001, vae:0.019, IQA:4.361, Connector:0.077, model:1.335, lr:6.626e-07, loss:0.180loss:0.2785, grad_norm:0.4160 | |
| 2025-10-25 02:13:04 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15101 | Local Step: 3000 // 12101, total_eta: 84 days, 5:26:08, epoch_eta:15:23:42, time: all:6.087, data:0.001, vae:0.018, IQA:4.287, Connector:0.067, model:1.324, lr:6.668e-07, loss:0.027loss:0.3211, grad_norm:0.4860 | |
| 2025-10-25 02:23:09 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15201 | Local Step: 3100 // 12101, total_eta: 84 days, 4:49:09, epoch_eta:15:13:21, time: all:6.048, data:0.001, vae:0.018, IQA:4.268, Connector:0.070, model:1.286, lr:6.718e-07, loss:1.055loss:0.3137, grad_norm:0.9626 | |
| 2025-10-25 02:33:11 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15301 | Local Step: 3200 // 12101, total_eta: 84 days, 3:57:40, epoch_eta:15:02:53, time: all:6.022, data:0.001, vae:0.018, IQA:4.265, Connector:0.067, model:1.283, lr:6.760e-07, loss:0.426loss:0.2987, grad_norm:0.4053 | |
| 2025-10-25 02:43:21 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15401 | Local Step: 3300 // 12101, total_eta: 84 days, 3:53:28, epoch_eta:14:52:47, time: all:6.096, data:0.001, vae:0.018, IQA:4.339, Connector:0.068, model:1.275, lr:6.802e-07, loss:0.629loss:0.3054, grad_norm:0.5026 | |
| 2025-10-25 02:53:31 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15501 | Local Step: 3400 // 12101, total_eta: 84 days, 3:49:46, epoch_eta:14:42:42, time: all:6.098, data:0.001, vae:0.017, IQA:4.334, Connector:0.069, model:1.278, lr:6.845e-07, loss:0.118loss:0.3134, grad_norm:1.3527 | |
| 2025-10-25 03:03:36 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15601 | Local Step: 3500 // 12101, total_eta: 84 days, 3:20:38, epoch_eta:14:32:25, time: all:6.053, data:0.001, vae:0.018, IQA:4.315, Connector:0.067, model:1.256, lr:6.894e-07, loss:0.547loss:0.3000, grad_norm:0.3682 | |
| 2025-10-25 03:13:44 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15701 | Local Step: 3600 // 12101, total_eta: 84 days, 3:06:56, epoch_eta:14:22:15, time: all:6.079, data:0.001, vae:0.018, IQA:4.359, Connector:0.070, model:1.227, lr:6.937e-07, loss:0.602loss:0.3021, grad_norm:0.4710 | |
| 2025-10-25 03:23:49 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15801 | Local Step: 3700 // 12101, total_eta: 84 days, 2:38:41, epoch_eta:14:11:58, time: all:6.052, data:0.001, vae:0.017, IQA:4.269, Connector:0.069, model:1.301, lr:6.979e-07, loss:0.260loss:0.3074, grad_norm:0.4256 | |
| 2025-10-25 03:33:56 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 15901 | Local Step: 3800 // 12101, total_eta: 84 days, 2:23:03, epoch_eta:14:01:48, time: all:6.074, data:0.001, vae:0.018, IQA:4.292, Connector:0.070, model:1.289, lr:7.022e-07, loss:0.436loss:0.3173, grad_norm:0.5046 | |
| 2025-10-25 03:44:17 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16001 | Local Step: 3900 // 12101, total_eta: 84 days, 3:14:05, epoch_eta:13:52:04, time: all:6.204, data:0.001, vae:0.018, IQA:4.223, Connector:0.079, model:1.400, lr:7.071e-07, loss:0.217loss:0.3165, grad_norm:0.5167 | |
| 2025-10-25 03:54:22 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16101 | Local Step: 4000 // 12101, total_eta: 84 days, 2:44:29, epoch_eta:13:41:48, time: all:6.049, data:0.001, vae:0.018, IQA:4.308, Connector:0.069, model:1.253, lr:7.113e-07, loss:0.385loss:0.3147, grad_norm:0.8734 | |
| 2025-10-25 04:04:32 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16201 | Local Step: 4100 // 12101, total_eta: 84 days, 2:40:27, epoch_eta:13:31:41, time: all:6.099, data:0.001, vae:0.018, IQA:4.358, Connector:0.071, model:1.240, lr:7.156e-07, loss:0.210loss:0.3137, grad_norm:0.4658 | |
| 2025-10-25 04:14:37 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16301 | Local Step: 4200 // 12101, total_eta: 84 days, 2:13:29, epoch_eta:13:21:26, time: all:6.051, data:0.001, vae:0.018, IQA:4.337, Connector:0.067, model:1.232, lr:7.198e-07, loss:0.367loss:0.3016, grad_norm:0.3242 | |
| 2025-10-25 04:24:44 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16401 | Local Step: 4300 // 12101, total_eta: 84 days, 1:57:04, epoch_eta:13:11:15, time: all:6.073, data:0.001, vae:0.018, IQA:4.320, Connector:0.068, model:1.270, lr:7.248e-07, loss:0.629loss:0.2937, grad_norm:0.4199 | |
| 2025-10-25 04:34:53 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16501 | Local Step: 4400 // 12101, total_eta: 84 days, 1:46:24, epoch_eta:13:01:06, time: all:6.085, data:0.001, vae:0.018, IQA:4.341, Connector:0.067, model:1.260, lr:7.290e-07, loss:0.242loss:0.3076, grad_norm:0.8405 | |
| 2025-10-25 04:44:56 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16601 | Local Step: 4500 // 12101, total_eta: 84 days, 1:15:21, epoch_eta:12:50:50, time: all:6.038, data:0.001, vae:0.018, IQA:4.300, Connector:0.068, model:1.268, lr:7.333e-07, loss:0.092loss:0.3119, grad_norm:1.0091 | |
| 2025-10-25 04:55:02 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16701 | Local Step: 4600 // 12101, total_eta: 84 days, 0:52:47, epoch_eta:12:40:36, time: all:6.056, data:0.001, vae:0.018, IQA:4.224, Connector:0.071, model:1.338, lr:7.375e-07, loss:0.449loss:0.3021, grad_norm:0.5537 | |
| 2025-10-25 05:05:10 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16801 | Local Step: 4700 // 12101, total_eta: 84 days, 0:41:39, epoch_eta:12:30:28, time: all:6.082, data:0.001, vae:0.017, IQA:4.321, Connector:0.067, model:1.268, lr:7.425e-07, loss:0.193loss:0.3049, grad_norm:0.6496 | |
| 2025-10-25 05:15:23 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 16901 | Local Step: 4800 // 12101, total_eta: 84 days, 0:50:00, epoch_eta:12:20:26, time: all:6.129, data:0.001, vae:0.018, IQA:4.317, Connector:0.068, model:1.329, lr:7.467e-07, loss:0.135loss:0.3291, grad_norm:0.5083 | |
| 2025-10-25 05:25:47 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17001 | Local Step: 4900 // 12101, total_eta: 84 days, 1:41:10, epoch_eta:12:10:40, time: all:6.236, data:0.001, vae:0.019, IQA:4.345, Connector:0.077, model:1.327, lr:7.509e-07, loss:0.256loss:0.3140, grad_norm:0.5534 | |
| 2025-10-25 05:35:46 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17101 | Local Step: 5000 // 12101, total_eta: 84 days, 0:54:28, epoch_eta:12:00:18, time: all:5.996, data:0.001, vae:0.018, IQA:4.251, Connector:0.068, model:1.280, lr:7.552e-07, loss:1.094loss:0.3044, grad_norm:0.7109 | |
| 2025-10-25 05:45:49 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17201 | Local Step: 5100 // 12101, total_eta: 84 days, 0:22:06, epoch_eta:11:50:01, time: all:6.029, data:0.001, vae:0.018, IQA:4.265, Connector:0.072, model:1.280, lr:7.601e-07, loss:0.133loss:0.3205, grad_norm:0.4282 | |
| 2025-10-25 05:55:52 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17301 | Local Step: 5200 // 12101, total_eta: 83 days, 23:48:57, epoch_eta:11:39:45, time: all:6.025, data:0.001, vae:0.018, IQA:4.239, Connector:0.070, model:1.319, lr:7.644e-07, loss:0.209loss:0.3126, grad_norm:0.6872 | |
| 2025-10-25 06:05:53 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17401 | Local Step: 5300 // 12101, total_eta: 83 days, 23:13:58, epoch_eta:11:29:28, time: all:6.018, data:0.001, vae:0.018, IQA:4.261, Connector:0.066, model:1.291, lr:7.686e-07, loss:0.189loss:0.3208, grad_norm:0.3836 | |
| 2025-10-25 06:15:57 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17501 | Local Step: 5400 // 12101, total_eta: 83 days, 22:45:14, epoch_eta:11:19:13, time: all:6.032, data:0.001, vae:0.018, IQA:4.347, Connector:0.066, model:1.227, lr:7.729e-07, loss:0.214loss:0.2941, grad_norm:0.4766 | |
| 2025-10-25 06:26:00 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17601 | Local Step: 5500 // 12101, total_eta: 83 days, 22:18:20, epoch_eta:11:09:00, time: all:6.035, data:0.001, vae:0.018, IQA:4.308, Connector:0.068, model:1.263, lr:7.778e-07, loss:0.166loss:0.3026, grad_norm:0.4858 | |
| 2025-10-25 06:36:06 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17701 | Local Step: 5600 // 12101, total_eta: 83 days, 21:58:33, epoch_eta:10:58:48, time: all:6.054, data:0.001, vae:0.019, IQA:4.301, Connector:0.067, model:1.293, lr:7.821e-07, loss:0.067loss:0.3063, grad_norm:0.4131 | |
| 2025-10-25 06:46:10 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17801 | Local Step: 5700 // 12101, total_eta: 83 days, 21:37:12, epoch_eta:10:48:37, time: all:6.048, data:0.001, vae:0.018, IQA:4.291, Connector:0.071, model:1.291, lr:7.863e-07, loss:0.598loss:0.3132, grad_norm:0.6507 | |
| 2025-10-25 06:56:08 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 17901 | Local Step: 5800 // 12101, total_eta: 83 days, 20:52:04, epoch_eta:10:38:18, time: all:5.978, data:0.001, vae:0.019, IQA:4.312, Connector:0.066, model:1.222, lr:7.905e-07, loss:0.293loss:0.3228, grad_norm:0.7523 | |
| 2025-10-25 07:06:33 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18001 | Local Step: 5900 // 12101, total_eta: 83 days, 21:39:35, epoch_eta:10:28:28, time: all:6.249, data:0.001, vae:0.019, IQA:4.349, Connector:0.076, model:1.363, lr:7.955e-07, loss:0.225loss:0.3140, grad_norm:0.5237 | |
| 2025-10-25 07:16:33 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18101 | Local Step: 6000 // 12101, total_eta: 83 days, 21:03:39, epoch_eta:10:18:12, time: all:6.003, data:0.001, vae:0.019, IQA:4.220, Connector:0.068, model:1.311, lr:7.997e-07, loss:0.232loss:0.2985, grad_norm:0.4222 | |
| 2025-10-25 07:26:34 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18201 | Local Step: 6100 // 12101, total_eta: 83 days, 20:28:20, epoch_eta:10:07:56, time: all:6.002, data:0.001, vae:0.018, IQA:4.268, Connector:0.070, model:1.264, lr:8.040e-07, loss:0.527loss:0.3018, grad_norm:0.9710 | |
| 2025-10-25 07:36:36 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18301 | Local Step: 6200 // 12101, total_eta: 83 days, 20:00:23, epoch_eta:9:57:43, time: all:6.023, data:0.001, vae:0.018, IQA:4.260, Connector:0.070, model:1.280, lr:8.082e-07, loss:0.258loss:0.2994, grad_norm:0.5078 | |
| 2025-10-25 07:46:41 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18401 | Local Step: 6300 // 12101, total_eta: 83 days, 19:41:30, epoch_eta:9:47:33, time: all:6.050, data:0.001, vae:0.018, IQA:4.238, Connector:0.072, model:1.321, lr:8.132e-07, loss:0.383loss:0.2980, grad_norm:0.4159 | |
| 2025-10-25 07:56:34 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18501 | Local Step: 6400 // 12101, total_eta: 83 days, 18:45:52, epoch_eta:9:37:12, time: all:5.930, data:0.001, vae:0.018, IQA:4.219, Connector:0.071, model:1.231, lr:8.174e-07, loss:0.992loss:0.3027, grad_norm:0.3802 | |
| 2025-10-25 08:06:34 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18601 | Local Step: 6500 // 12101, total_eta: 83 days, 18:12:00, epoch_eta:9:26:58, time: all:5.997, data:0.001, vae:0.018, IQA:4.277, Connector:0.070, model:1.246, lr:8.217e-07, loss:0.010loss:0.3008, grad_norm:0.4691 | |
| 2025-10-25 08:16:37 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18701 | Local Step: 6600 // 12101, total_eta: 83 days, 17:50:26, epoch_eta:9:16:47, time: all:6.036, data:0.001, vae:0.018, IQA:4.331, Connector:0.069, model:1.236, lr:8.259e-07, loss:0.165loss:0.3144, grad_norm:0.4870 | |
| 2025-10-25 08:26:39 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18801 | Local Step: 6700 // 12101, total_eta: 83 days, 17:24:55, epoch_eta:9:06:36, time: all:6.021, data:0.001, vae:0.018, IQA:4.283, Connector:0.066, model:1.273, lr:8.309e-07, loss:0.250loss:0.2904, grad_norm:0.4272 | |
| 2025-10-25 08:36:49 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 18901 | Local Step: 6800 // 12101, total_eta: 83 days, 17:20:30, epoch_eta:8:56:30, time: all:6.092, data:0.001, vae:0.018, IQA:4.336, Connector:0.070, model:1.273, lr:8.351e-07, loss:0.730loss:0.3072, grad_norm:0.5270 | |
| 2025-10-25 08:47:02 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19001 | Local Step: 6900 // 12101, total_eta: 83 days, 17:29:21, epoch_eta:8:46:28, time: all:6.138, data:0.001, vae:0.019, IQA:4.220, Connector:0.077, model:1.347, lr:8.393e-07, loss:0.688loss:0.2923, grad_norm:0.4762 | |
| 2025-10-25 08:57:05 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19101 | Local Step: 7000 // 12101, total_eta: 83 days, 17:06:34, epoch_eta:8:36:17, time: all:6.029, data:0.001, vae:0.018, IQA:4.288, Connector:0.068, model:1.272, lr:8.436e-07, loss:0.357loss:0.3054, grad_norm:1.2257 | |
| 2025-10-25 09:07:07 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19201 | Local Step: 7100 // 12101, total_eta: 83 days, 16:39:48, epoch_eta:8:26:06, time: all:6.013, data:0.001, vae:0.018, IQA:4.305, Connector:0.067, model:1.246, lr:8.485e-07, loss:0.281loss:0.3201, grad_norm:0.6247 | |
| 2025-10-25 09:17:09 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19301 | Local Step: 7200 // 12101, total_eta: 83 days, 16:16:14, epoch_eta:8:15:55, time: all:6.023, data:0.001, vae:0.018, IQA:4.292, Connector:0.069, model:1.258, lr:8.528e-07, loss:0.225loss:0.2971, grad_norm:0.7204 | |
| 2025-10-25 09:27:13 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19401 | Local Step: 7300 // 12101, total_eta: 83 days, 15:58:24, epoch_eta:8:05:46, time: all:6.043, data:0.001, vae:0.018, IQA:4.280, Connector:0.067, model:1.295, lr:8.570e-07, loss:0.229loss:0.3046, grad_norm:0.5117 | |
| 2025-10-25 09:37:08 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19501 | Local Step: 7400 // 12101, total_eta: 83 days, 15:16:14, epoch_eta:7:55:32, time: all:5.952, data:0.001, vae:0.018, IQA:4.183, Connector:0.068, model:1.290, lr:8.613e-07, loss:0.406loss:0.3037, grad_norm:0.5692 | |
| 2025-10-25 09:47:12 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19601 | Local Step: 7500 // 12101, total_eta: 83 days, 14:58:13, epoch_eta:7:45:23, time: all:6.039, data:0.001, vae:0.018, IQA:4.339, Connector:0.067, model:1.238, lr:8.662e-07, loss:0.178loss:0.3179, grad_norm:0.3714 | |
| 2025-10-25 09:57:11 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19701 | Local Step: 7600 // 12101, total_eta: 83 days, 14:25:30, epoch_eta:7:35:11, time: all:5.982, data:0.001, vae:0.018, IQA:4.183, Connector:0.071, model:1.319, lr:8.704e-07, loss:0.492loss:0.3086, grad_norm:0.4984 | |
| 2025-10-25 10:07:11 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19801 | Local Step: 7700 // 12101, total_eta: 83 days, 13:59:34, epoch_eta:7:25:01, time: all:6.006, data:0.001, vae:0.018, IQA:4.265, Connector:0.069, model:1.268, lr:8.747e-07, loss:0.014loss:0.3093, grad_norm:0.4801 | |
| 2025-10-25 10:17:19 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 19901 | Local Step: 7800 // 12101, total_eta: 83 days, 13:51:33, epoch_eta:7:14:54, time: all:6.075, data:0.001, vae:0.018, IQA:4.283, Connector:0.070, model:1.298, lr:8.789e-07, loss:0.225loss:0.3144, grad_norm:0.8167 | |
| 2025-10-25 10:27:21 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 2 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_2_step_20000_weight.pth. | |
| 2025-10-25 10:27:48 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 2 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_2_step_20000.pth. | |
| 2025-10-25 10:28:17 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20001 | Local Step: 7900 // 12101, total_eta: 83 days, 15:51:21, epoch_eta:7:05:15, time: all:6.585, data:0.001, vae:0.019, IQA:4.264, Connector:0.077, model:1.427, lr:8.839e-07, loss:0.162loss:0.3053, grad_norm:0.5240 | |
| 2025-10-25 10:38:20 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20101 | Local Step: 8000 // 12101, total_eta: 83 days, 15:30:56, epoch_eta:6:55:06, time: all:6.032, data:0.001, vae:0.018, IQA:4.260, Connector:0.069, model:1.279, lr:8.881e-07, loss:0.154loss:0.2973, grad_norm:0.5788 | |
| 2025-10-25 10:48:24 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20201 | Local Step: 8100 // 12101, total_eta: 83 days, 15:10:48, epoch_eta:6:44:56, time: all:6.032, data:0.001, vae:0.018, IQA:4.321, Connector:0.066, model:1.232, lr:8.924e-07, loss:0.042loss:0.3102, grad_norm:0.8789 | |
| 2025-10-25 10:58:32 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20301 | Local Step: 8200 // 12101, total_eta: 83 days, 15:04:08, epoch_eta:6:34:50, time: all:6.087, data:0.001, vae:0.018, IQA:4.343, Connector:0.068, model:1.270, lr:8.966e-07, loss:0.207loss:0.2949, grad_norm:0.6559 | |
| 2025-10-25 11:08:36 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20401 | Local Step: 8300 // 12101, total_eta: 83 days, 14:44:34, epoch_eta:6:24:41, time: all:6.033, data:0.001, vae:0.018, IQA:4.252, Connector:0.069, model:1.284, lr:9.016e-07, loss:0.178loss:0.3009, grad_norm:0.4869 | |
| 2025-10-25 11:18:40 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20501 | Local Step: 8400 // 12101, total_eta: 83 days, 14:28:21, epoch_eta:6:14:32, time: all:6.047, data:0.001, vae:0.018, IQA:4.268, Connector:0.068, model:1.295, lr:9.058e-07, loss:0.181loss:0.3024, grad_norm:0.4733 | |
| 2025-10-25 11:28:42 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20601 | Local Step: 8500 // 12101, total_eta: 83 days, 14:06:00, epoch_eta:6:04:23, time: all:6.020, data:0.001, vae:0.018, IQA:4.274, Connector:0.068, model:1.272, lr:9.100e-07, loss:0.069loss:0.2946, grad_norm:0.4460 | |
| 2025-10-25 11:38:54 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20701 | Local Step: 8600 // 12101, total_eta: 83 days, 14:05:54, epoch_eta:5:54:17, time: all:6.115, data:0.001, vae:0.018, IQA:4.329, Connector:0.067, model:1.298, lr:9.143e-07, loss:0.256loss:0.3013, grad_norm:0.8125 | |
| 2025-10-25 11:49:00 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20801 | Local Step: 8700 // 12101, total_eta: 83 days, 13:52:38, epoch_eta:5:44:10, time: all:6.058, data:0.001, vae:0.018, IQA:4.299, Connector:0.068, model:1.269, lr:9.192e-07, loss:0.035loss:0.3145, grad_norm:0.5949 | |
| 2025-10-25 11:59:04 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 20901 | Local Step: 8800 // 12101, total_eta: 83 days, 13:36:31, epoch_eta:5:34:01, time: all:6.045, data:0.001, vae:0.018, IQA:4.284, Connector:0.069, model:1.275, lr:9.235e-07, loss:0.164loss:0.2937, grad_norm:0.5273 | |
| 2025-10-25 12:09:24 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21001 | Local Step: 8900 // 12101, total_eta: 83 days, 13:55:30, epoch_eta:5:23:59, time: all:6.202, data:0.001, vae:0.018, IQA:4.353, Connector:0.076, model:1.291, lr:9.277e-07, loss:0.562loss:0.3191, grad_norm:0.3789 | |
| 2025-10-25 12:19:31 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21101 | Local Step: 9000 // 12101, total_eta: 83 days, 13:43:57, epoch_eta:5:13:51, time: all:6.066, data:0.001, vae:0.018, IQA:4.273, Connector:0.070, model:1.300, lr:9.320e-07, loss:0.443loss:0.2921, grad_norm:0.4362 | |
| 2025-10-25 12:29:40 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21201 | Local Step: 9100 // 12101, total_eta: 83 days, 13:38:16, epoch_eta:5:03:45, time: all:6.093, data:0.001, vae:0.018, IQA:4.321, Connector:0.066, model:1.285, lr:9.369e-07, loss:0.320loss:0.2864, grad_norm:0.4842 | |
| 2025-10-25 12:39:49 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21301 | Local Step: 9200 // 12101, total_eta: 83 days, 13:30:35, epoch_eta:4:53:38, time: all:6.084, data:0.001, vae:0.018, IQA:4.308, Connector:0.068, model:1.298, lr:9.412e-07, loss:0.412loss:0.3038, grad_norm:0.5495 | |
| 2025-10-25 12:49:58 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21401 | Local Step: 9300 // 12101, total_eta: 83 days, 13:24:21, epoch_eta:4:43:31, time: all:6.091, data:0.001, vae:0.018, IQA:4.301, Connector:0.070, model:1.301, lr:9.454e-07, loss:0.239loss:0.3127, grad_norm:0.4495 | |
| 2025-10-25 12:59:59 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21501 | Local Step: 9400 // 12101, total_eta: 83 days, 13:00:46, epoch_eta:4:33:22, time: all:6.010, data:0.001, vae:0.018, IQA:4.222, Connector:0.066, model:1.295, lr:9.496e-07, loss:0.480loss:0.3136, grad_norm:0.5928 | |
| 2025-10-25 13:10:07 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21601 | Local Step: 9500 // 12101, total_eta: 83 days, 12:52:31, epoch_eta:4:23:15, time: all:6.082, data:0.001, vae:0.018, IQA:4.320, Connector:0.072, model:1.269, lr:9.546e-07, loss:0.193loss:0.3086, grad_norm:0.5982 | |
| 2025-10-25 13:20:17 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21701 | Local Step: 9600 // 12101, total_eta: 83 days, 12:48:12, epoch_eta:4:13:08, time: all:6.101, data:0.001, vae:0.018, IQA:4.351, Connector:0.068, model:1.264, lr:9.588e-07, loss:0.010loss:0.2869, grad_norm:0.3934 | |
| 2025-10-25 13:30:26 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21801 | Local Step: 9700 // 12101, total_eta: 83 days, 12:41:11, epoch_eta:4:03:01, time: all:6.088, data:0.001, vae:0.018, IQA:4.355, Connector:0.068, model:1.253, lr:9.631e-07, loss:0.197loss:0.3020, grad_norm:0.4679 | |
| 2025-10-25 13:40:33 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 21901 | Local Step: 9800 // 12101, total_eta: 83 days, 12:30:29, epoch_eta:3:52:54, time: all:6.070, data:0.001, vae:0.019, IQA:4.221, Connector:0.072, model:1.335, lr:9.673e-07, loss:0.227loss:0.3101, grad_norm:0.3760 | |
| 2025-10-25 13:50:55 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22001 | Local Step: 9900 // 12101, total_eta: 83 days, 12:49:46, epoch_eta:3:42:50, time: all:6.220, data:0.001, vae:0.019, IQA:4.342, Connector:0.076, model:1.314, lr:9.723e-07, loss:0.240loss:0.2853, grad_norm:0.6929 | |
| 2025-10-25 14:00:59 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22101 | Local Step: 10000 // 12101, total_eta: 83 days, 12:32:41, epoch_eta:3:32:42, time: all:6.040, data:0.001, vae:0.018, IQA:4.295, Connector:0.067, model:1.268, lr:9.765e-07, loss:0.111loss:0.3066, grad_norm:0.5371 | |
| 2025-10-25 14:11:02 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22201 | Local Step: 10100 // 12101, total_eta: 83 days, 12:14:24, epoch_eta:3:22:34, time: all:6.033, data:0.001, vae:0.018, IQA:4.300, Connector:0.070, model:1.244, lr:9.808e-07, loss:0.186loss:0.3038, grad_norm:0.8044 | |
| 2025-10-25 14:21:04 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22301 | Local Step: 10200 // 12101, total_eta: 83 days, 11:52:25, epoch_eta:3:12:25, time: all:6.013, data:0.001, vae:0.018, IQA:4.300, Connector:0.069, model:1.238, lr:9.850e-07, loss:1.828loss:0.2938, grad_norm:0.5602 | |
| 2025-10-25 14:31:09 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22401 | Local Step: 10300 // 12101, total_eta: 83 days, 11:38:10, epoch_eta:3:02:17, time: all:6.052, data:0.001, vae:0.018, IQA:4.291, Connector:0.068, model:1.263, lr:9.899e-07, loss:0.301loss:0.3076, grad_norm:0.4051 | |
| 2025-10-25 14:41:10 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22501 | Local Step: 10400 // 12101, total_eta: 83 days, 11:16:27, epoch_eta:2:52:09, time: all:6.012, data:0.001, vae:0.018, IQA:4.287, Connector:0.067, model:1.240, lr:9.942e-07, loss:0.287loss:0.2981, grad_norm:0.6130 | |
| 2025-10-25 14:51:17 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22601 | Local Step: 10500 // 12101, total_eta: 83 days, 11:05:02, epoch_eta:2:42:02, time: all:6.066, data:0.001, vae:0.018, IQA:4.262, Connector:0.069, model:1.312, lr:9.984e-07, loss:0.598loss:0.3140, grad_norm:0.5771 | |
| 2025-10-25 15:01:19 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22701 | Local Step: 10600 // 12101, total_eta: 83 days, 10:46:30, epoch_eta:2:31:54, time: all:6.027, data:0.001, vae:0.018, IQA:4.263, Connector:0.068, model:1.288, lr:1.003e-06, loss:0.220loss:0.3035, grad_norm:0.3771 | |
| 2025-10-25 15:11:18 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22801 | Local Step: 10700 // 12101, total_eta: 83 days, 10:20:49, epoch_eta:2:21:45, time: all:5.988, data:0.001, vae:0.018, IQA:4.212, Connector:0.070, model:1.273, lr:1.008e-06, loss:1.172loss:0.3074, grad_norm:0.4561 | |
| 2025-10-25 15:21:20 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 22901 | Local Step: 10800 // 12101, total_eta: 83 days, 10:01:32, epoch_eta:2:11:38, time: all:6.021, data:0.001, vae:0.018, IQA:4.337, Connector:0.069, model:1.207, lr:1.012e-06, loss:0.229loss:0.2898, grad_norm:1.0046 | |
| 2025-10-25 15:31:44 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23001 | Local Step: 10900 // 12101, total_eta: 83 days, 10:20:52, epoch_eta:2:01:32, time: all:6.233, data:0.001, vae:0.018, IQA:4.334, Connector:0.076, model:1.309, lr:1.016e-06, loss:0.457loss:0.3056, grad_norm:1.1322 | |
| 2025-10-25 15:41:41 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23101 | Local Step: 11000 // 12101, total_eta: 83 days, 9:53:27, epoch_eta:1:51:24, time: all:5.976, data:0.001, vae:0.019, IQA:4.280, Connector:0.069, model:1.215, lr:1.020e-06, loss:0.111loss:0.3047, grad_norm:0.4230 | |
| 2025-10-25 15:51:46 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23201 | Local Step: 11100 // 12101, total_eta: 83 days, 9:39:31, epoch_eta:1:41:17, time: all:6.050, data:0.001, vae:0.018, IQA:4.265, Connector:0.068, model:1.299, lr:1.025e-06, loss:0.145loss:0.2996, grad_norm:0.4997 | |
| 2025-10-25 16:01:55 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23301 | Local Step: 11200 // 12101, total_eta: 83 days, 9:32:07, epoch_eta:1:31:10, time: all:6.087, data:0.001, vae:0.018, IQA:4.324, Connector:0.067, model:1.284, lr:1.030e-06, loss:0.453loss:0.3082, grad_norm:0.4539 | |
| 2025-10-25 16:12:02 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23401 | Local Step: 11300 // 12101, total_eta: 83 days, 9:21:50, epoch_eta:1:21:03, time: all:6.070, data:0.001, vae:0.018, IQA:4.293, Connector:0.070, model:1.280, lr:1.034e-06, loss:0.145loss:0.2962, grad_norm:0.3590 | |
| 2025-10-25 16:22:07 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23501 | Local Step: 11400 // 12101, total_eta: 83 days, 9:07:48, epoch_eta:1:10:55, time: all:6.049, data:0.001, vae:0.018, IQA:4.275, Connector:0.070, model:1.301, lr:1.038e-06, loss:0.324loss:0.2945, grad_norm:0.5439 | |
| 2025-10-25 16:32:12 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23601 | Local Step: 11500 // 12101, total_eta: 83 days, 8:54:54, epoch_eta:1:00:48, time: all:6.055, data:0.001, vae:0.018, IQA:4.340, Connector:0.069, model:1.241, lr:1.043e-06, loss:0.007loss:0.2900, grad_norm:0.8203 | |
| 2025-10-25 16:42:17 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23701 | Local Step: 11600 // 12101, total_eta: 83 days, 8:41:33, epoch_eta:0:50:41, time: all:6.052, data:0.001, vae:0.018, IQA:4.321, Connector:0.070, model:1.255, lr:1.047e-06, loss:0.422loss:0.2705, grad_norm:0.5060 | |
| 2025-10-25 16:52:21 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23801 | Local Step: 11700 // 12101, total_eta: 83 days, 8:25:04, epoch_eta:0:40:34, time: all:6.033, data:0.001, vae:0.019, IQA:4.252, Connector:0.069, model:1.310, lr:1.051e-06, loss:0.383loss:0.3073, grad_norm:0.5365 | |
| 2025-10-25 17:02:21 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 23901 | Local Step: 11800 // 12101, total_eta: 83 days, 8:03:06, epoch_eta:0:30:27, time: all:6.000, data:0.001, vae:0.018, IQA:4.299, Connector:0.068, model:1.233, lr:1.056e-06, loss:0.006loss:0.2921, grad_norm:0.5316 | |
| 2025-10-25 17:12:44 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 24001 | Local Step: 11900 // 12101, total_eta: 83 days, 8:20:13, epoch_eta:0:20:20, time: all:6.234, data:0.001, vae:0.020, IQA:4.302, Connector:0.078, model:1.380, lr:1.061e-06, loss:0.025loss:0.3014, grad_norm:0.5084 | |
| 2025-10-25 17:22:46 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 24101 | Local Step: 12000 // 12101, total_eta: 83 days, 8:00:59, epoch_eta:0:10:13, time: all:6.016, data:0.001, vae:0.018, IQA:4.335, Connector:0.067, model:1.227, lr:1.065e-06, loss:0.398loss:0.2961, grad_norm:0.4818 | |
| 2025-10-25 17:32:50 - [1m[Sana][0m - INFO - Epoch: 2 | Global Step: 24201 | Local Step: 12100 // 12101, total_eta: 83 days, 7:45:41, epoch_eta:0:00:06, time: all:6.039, data:0.001, vae:0.018, IQA:4.315, Connector:0.069, model:1.260, lr:1.069e-06, loss:0.209loss:0.2774, grad_norm:0.3773 | |
| 2025-10-25 17:33:01 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 2 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_2_step_24203_weight.pth. | |
| 2025-10-25 17:33:30 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 2 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_2_step_24203.pth. | |
| 2025-10-25 17:33:38 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24203 | Local Step: 1 // 12101, total_eta: 115 days, 5:26:52, epoch_eta:1 day, 4:12:59, time: all:0.084, data:0.016, vae:0.000, IQA:0.051, Connector:0.001, model:0.010, lr:1.069e-06, loss:0.213loss:0.1802 | |
| 2025-10-25 17:43:50 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24302 | Local Step: 100 // 12101, total_eta: 85 days, 2:44:23, epoch_eta:20:40:25, time: all:6.118, data:0.001, vae:0.019, IQA:4.317, Connector:0.075, model:1.270, lr:1.073e-06, loss:0.134loss:0.2925, grad_norm:0.9007 | |
| 2025-10-25 17:53:51 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24402 | Local Step: 200 // 12101, total_eta: 83 days, 17:56:19, epoch_eta:20:10:26, time: all:6.003, data:0.001, vae:0.018, IQA:4.207, Connector:0.070, model:1.324, lr:1.078e-06, loss:0.199loss:0.2858, grad_norm:0.7552 | |
| 2025-10-25 18:03:54 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24502 | Local Step: 300 // 12101, total_eta: 83 days, 10:31:55, epoch_eta:19:55:56, time: all:6.037, data:0.001, vae:0.018, IQA:4.280, Connector:0.070, model:1.286, lr:1.083e-06, loss:0.015loss:0.3072, grad_norm:0.5443 | |
| 2025-10-25 18:13:57 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24602 | Local Step: 400 // 12101, total_eta: 83 days, 6:15:56, epoch_eta:19:43:23, time: all:6.031, data:0.001, vae:0.019, IQA:4.304, Connector:0.068, model:1.260, lr:1.087e-06, loss:0.309loss:0.3011, grad_norm:0.6131 | |
| 2025-10-25 18:23:56 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24702 | Local Step: 500 // 12101, total_eta: 83 days, 0:52:25, epoch_eta:19:30:12, time: all:5.989, data:0.001, vae:0.019, IQA:4.250, Connector:0.070, model:1.276, lr:1.091e-06, loss:0.434loss:0.2909, grad_norm:0.4235 | |
| 2025-10-25 18:34:03 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24802 | Local Step: 600 // 12101, total_eta: 83 days, 1:47:30, epoch_eta:19:20:45, time: all:6.072, data:0.001, vae:0.018, IQA:4.330, Connector:0.069, model:1.269, lr:1.096e-06, loss:0.199loss:0.2853, grad_norm:0.7012 | |
| 2025-10-25 18:44:03 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 24902 | Local Step: 700 // 12101, total_eta: 82 days, 22:34:51, epoch_eta:19:08:54, time: all:5.991, data:0.001, vae:0.019, IQA:4.290, Connector:0.069, model:1.242, lr:1.100e-06, loss:0.137loss:0.3074, grad_norm:0.4948 | |
| 2025-10-25 18:53:59 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_25000_weight.pth. | |
| 2025-10-25 18:54:28 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_25000.pth. | |
| 2025-10-25 18:55:04 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25002 | Local Step: 800 // 12101, total_eta: 83 days, 21:52:56, epoch_eta:19:12:15, time: all:6.617, data:0.001, vae:0.019, IQA:4.470, Connector:0.076, model:1.259, lr:1.105e-06, loss:0.247loss:0.2970, grad_norm:0.5890 | |
| 2025-10-25 19:05:08 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25102 | Local Step: 900 // 12101, total_eta: 83 days, 18:40:34, epoch_eta:19:00:20, time: all:6.035, data:0.001, vae:0.018, IQA:4.309, Connector:0.066, model:1.272, lr:1.109e-06, loss:0.131loss:0.2958, grad_norm:0.4417 | |
| 2025-10-25 19:15:15 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25202 | Local Step: 1000 // 12101, total_eta: 83 days, 17:21:45, epoch_eta:18:49:30, time: all:6.074, data:0.001, vae:0.019, IQA:4.408, Connector:0.066, model:1.213, lr:1.114e-06, loss:0.103loss:0.2889, grad_norm:0.4941 | |
| 2025-10-25 19:25:17 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25302 | Local Step: 1100 // 12101, total_eta: 83 days, 14:43:46, epoch_eta:18:37:57, time: all:6.023, data:0.001, vae:0.018, IQA:4.270, Connector:0.070, model:1.291, lr:1.118e-06, loss:0.215loss:0.2966, grad_norm:0.5853 | |
| 2025-10-25 19:35:22 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25402 | Local Step: 1200 // 12101, total_eta: 83 days, 13:07:36, epoch_eta:18:27:00, time: all:6.045, data:0.001, vae:0.018, IQA:4.334, Connector:0.068, model:1.237, lr:1.122e-06, loss:0.008loss:0.2805, grad_norm:0.3646 | |
| 2025-10-25 19:45:23 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25502 | Local Step: 1300 // 12101, total_eta: 83 days, 10:51:35, epoch_eta:18:15:42, time: all:6.010, data:0.001, vae:0.018, IQA:4.275, Connector:0.068, model:1.277, lr:1.126e-06, loss:0.238loss:0.2981, grad_norm:0.4209 | |
| 2025-10-25 19:55:33 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25602 | Local Step: 1400 // 12101, total_eta: 83 days, 11:04:24, epoch_eta:18:05:46, time: all:6.103, data:0.001, vae:0.018, IQA:4.391, Connector:0.068, model:1.244, lr:1.131e-06, loss:0.229loss:0.2956, grad_norm:0.5335 | |
| 2025-10-25 20:05:40 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25702 | Local Step: 1500 // 12101, total_eta: 83 days, 10:29:48, epoch_eta:17:55:24, time: all:6.069, data:0.001, vae:0.018, IQA:4.234, Connector:0.072, model:1.355, lr:1.136e-06, loss:0.102loss:0.2926, grad_norm:0.5319 | |
| 2025-10-25 20:15:41 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25802 | Local Step: 1600 // 12101, total_eta: 83 days, 8:44:38, epoch_eta:17:44:25, time: all:6.010, data:0.001, vae:0.019, IQA:4.322, Connector:0.068, model:1.227, lr:1.140e-06, loss:0.096loss:0.3095, grad_norm:0.4499 | |
| 2025-10-25 20:25:42 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 25902 | Local Step: 1700 // 12101, total_eta: 83 days, 7:09:13, epoch_eta:17:33:32, time: all:6.008, data:0.001, vae:0.018, IQA:4.299, Connector:0.068, model:1.253, lr:1.144e-06, loss:0.594loss:0.2820, grad_norm:0.3760 | |
| 2025-10-25 20:36:06 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26002 | Local Step: 1800 // 12101, total_eta: 83 days, 10:00:52, epoch_eta:17:24:59, time: all:6.243, data:0.001, vae:0.019, IQA:4.332, Connector:0.079, model:1.362, lr:1.149e-06, loss:0.270loss:0.2999, grad_norm:0.6861 | |
| 2025-10-25 20:46:13 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26102 | Local Step: 1900 // 12101, total_eta: 83 days, 9:29:07, epoch_eta:17:14:39, time: all:6.066, data:0.001, vae:0.018, IQA:4.262, Connector:0.069, model:1.325, lr:1.153e-06, loss:0.072loss:0.2872, grad_norm:0.3612 | |
| 2025-10-25 20:56:21 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26202 | Local Step: 2000 // 12101, total_eta: 83 days, 9:11:59, epoch_eta:17:04:27, time: all:6.079, data:0.001, vae:0.018, IQA:4.258, Connector:0.071, model:1.323, lr:1.158e-06, loss:0.137loss:0.3096, grad_norm:0.8366 | |
| 2025-10-25 21:06:27 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26302 | Local Step: 2100 // 12101, total_eta: 83 days, 8:43:38, epoch_eta:16:54:09, time: all:6.066, data:0.001, vae:0.018, IQA:4.270, Connector:0.069, model:1.300, lr:1.162e-06, loss:0.103loss:0.2928, grad_norm:0.3534 | |
| 2025-10-25 21:16:34 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26402 | Local Step: 2200 // 12101, total_eta: 83 days, 8:18:57, epoch_eta:16:43:53, time: all:6.068, data:0.001, vae:0.018, IQA:4.295, Connector:0.070, model:1.283, lr:1.167e-06, loss:0.129loss:0.2869, grad_norm:0.4643 | |
| 2025-10-25 21:26:42 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26502 | Local Step: 2300 // 12101, total_eta: 83 days, 8:04:11, epoch_eta:16:33:43, time: all:6.078, data:0.001, vae:0.018, IQA:4.283, Connector:0.068, model:1.294, lr:1.171e-06, loss:1.078loss:0.3128, grad_norm:1.1243 | |
| 2025-10-25 21:36:49 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26602 | Local Step: 2400 // 12101, total_eta: 83 days, 7:45:02, epoch_eta:16:23:30, time: all:6.072, data:0.001, vae:0.017, IQA:4.361, Connector:0.069, model:1.222, lr:1.175e-06, loss:0.139loss:0.3001, grad_norm:0.4538 | |
| 2025-10-25 21:46:49 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26702 | Local Step: 2500 // 12101, total_eta: 83 days, 6:26:55, epoch_eta:16:12:48, time: all:5.997, data:0.001, vae:0.018, IQA:4.256, Connector:0.070, model:1.236, lr:1.179e-06, loss:0.348loss:0.2857, grad_norm:0.3512 | |
| 2025-10-25 21:56:49 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26802 | Local Step: 2600 // 12101, total_eta: 83 days, 5:19:41, epoch_eta:16:02:13, time: all:6.004, data:0.001, vae:0.018, IQA:4.250, Connector:0.070, model:1.258, lr:1.184e-06, loss:0.221loss:0.3033, grad_norm:0.5056 | |
| 2025-10-25 22:07:01 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 26902 | Local Step: 2700 // 12101, total_eta: 83 days, 5:38:20, epoch_eta:15:52:19, time: all:6.116, data:0.001, vae:0.018, IQA:4.304, Connector:0.069, model:1.319, lr:1.189e-06, loss:0.135loss:0.3070, grad_norm:0.4800 | |
| 2025-10-25 22:17:18 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27002 | Local Step: 2800 // 12101, total_eta: 83 days, 6:34:58, epoch_eta:15:42:43, time: all:6.173, data:0.001, vae:0.019, IQA:4.210, Connector:0.076, model:1.384, lr:1.193e-06, loss:0.240loss:0.2804, grad_norm:0.4004 | |
| 2025-10-25 22:27:24 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27102 | Local Step: 2900 // 12101, total_eta: 83 days, 6:11:56, epoch_eta:15:32:29, time: all:6.062, data:0.001, vae:0.018, IQA:4.339, Connector:0.069, model:1.236, lr:1.197e-06, loss:0.414loss:0.2952, grad_norm:0.5993 | |
| 2025-10-25 22:37:29 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27202 | Local Step: 3000 // 12101, total_eta: 83 days, 5:42:23, epoch_eta:15:22:12, time: all:6.051, data:0.001, vae:0.018, IQA:4.325, Connector:0.070, model:1.239, lr:1.202e-06, loss:0.171loss:0.2988, grad_norm:0.4072 | |
| 2025-10-25 22:47:38 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27302 | Local Step: 3100 // 12101, total_eta: 83 days, 5:34:42, epoch_eta:15:12:05, time: all:6.084, data:0.001, vae:0.017, IQA:4.292, Connector:0.069, model:1.296, lr:1.206e-06, loss:0.115loss:0.2951, grad_norm:0.4111 | |
| 2025-10-25 22:57:47 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27402 | Local Step: 3200 // 12101, total_eta: 83 days, 5:30:00, epoch_eta:15:01:59, time: all:6.089, data:0.001, vae:0.017, IQA:4.287, Connector:0.069, model:1.303, lr:1.211e-06, loss:0.273loss:0.2985, grad_norm:0.7500 | |
| 2025-10-25 23:07:55 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27502 | Local Step: 3300 // 12101, total_eta: 83 days, 5:23:35, epoch_eta:14:51:53, time: all:6.086, data:0.001, vae:0.017, IQA:4.266, Connector:0.070, model:1.321, lr:1.215e-06, loss:0.129loss:0.2923, grad_norm:0.3657 | |
| 2025-10-25 23:18:00 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27602 | Local Step: 3400 // 12101, total_eta: 83 days, 4:52:55, epoch_eta:14:41:36, time: all:6.045, data:0.001, vae:0.018, IQA:4.259, Connector:0.071, model:1.294, lr:1.220e-06, loss:0.121loss:0.2797, grad_norm:0.7843 | |
| 2025-10-25 23:28:07 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27702 | Local Step: 3500 // 12101, total_eta: 83 days, 4:37:22, epoch_eta:14:31:25, time: all:6.070, data:0.001, vae:0.018, IQA:4.314, Connector:0.068, model:1.280, lr:1.224e-06, loss:0.301loss:0.2871, grad_norm:0.5127 | |
| 2025-10-25 23:38:11 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27802 | Local Step: 3600 // 12101, total_eta: 83 days, 4:08:08, epoch_eta:14:21:09, time: all:6.044, data:0.001, vae:0.018, IQA:4.295, Connector:0.071, model:1.255, lr:1.228e-06, loss:0.016loss:0.2840, grad_norm:0.4408 | |
| 2025-10-25 23:48:12 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 27902 | Local Step: 3700 // 12101, total_eta: 83 days, 3:20:29, epoch_eta:14:10:45, time: all:6.008, data:0.001, vae:0.018, IQA:4.253, Connector:0.070, model:1.255, lr:1.232e-06, loss:0.186loss:0.2970, grad_norm:1.2166 | |
| 2025-10-25 23:58:34 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28002 | Local Step: 3800 // 12101, total_eta: 83 days, 4:26:01, epoch_eta:14:01:10, time: all:6.222, data:0.001, vae:0.019, IQA:4.322, Connector:0.075, model:1.332, lr:1.237e-06, loss:0.777loss:0.2884, grad_norm:1.1380 | |
| 2025-10-26 00:08:44 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28102 | Local Step: 3900 // 12101, total_eta: 83 days, 4:23:49, epoch_eta:13:51:05, time: all:6.096, data:0.001, vae:0.018, IQA:4.390, Connector:0.065, model:1.223, lr:1.242e-06, loss:1.492loss:0.2940, grad_norm:0.3949 | |
| 2025-10-26 00:18:48 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28202 | Local Step: 4000 // 12101, total_eta: 83 days, 3:55:50, epoch_eta:13:40:50, time: all:6.044, data:0.001, vae:0.018, IQA:4.311, Connector:0.068, model:1.249, lr:1.246e-06, loss:0.270loss:0.3010, grad_norm:0.4408 | |
| 2025-10-26 00:29:00 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28302 | Local Step: 4100 // 12101, total_eta: 83 days, 4:03:37, epoch_eta:13:30:49, time: all:6.117, data:0.001, vae:0.018, IQA:4.338, Connector:0.071, model:1.285, lr:1.250e-06, loss:0.127loss:0.2835, grad_norm:0.2790 | |
| 2025-10-26 00:39:06 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28402 | Local Step: 4200 // 12101, total_eta: 83 days, 3:45:02, epoch_eta:13:20:37, time: all:6.062, data:0.001, vae:0.018, IQA:4.289, Connector:0.071, model:1.266, lr:1.255e-06, loss:0.016loss:0.2837, grad_norm:0.6270 | |
| 2025-10-26 00:49:08 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28502 | Local Step: 4300 // 12101, total_eta: 83 days, 3:05:50, epoch_eta:13:10:18, time: all:6.017, data:0.001, vae:0.018, IQA:4.286, Connector:0.068, model:1.240, lr:1.259e-06, loss:0.285loss:0.3012, grad_norm:0.7772 | |
| 2025-10-26 00:59:13 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28602 | Local Step: 4400 // 12101, total_eta: 83 days, 2:43:35, epoch_eta:13:00:05, time: all:6.051, data:0.001, vae:0.018, IQA:4.259, Connector:0.067, model:1.313, lr:1.264e-06, loss:0.508loss:0.2671, grad_norm:0.5197 | |
| 2025-10-26 01:09:16 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28702 | Local Step: 4500 // 12101, total_eta: 83 days, 2:11:11, epoch_eta:12:49:49, time: all:6.027, data:0.001, vae:0.018, IQA:4.259, Connector:0.069, model:1.283, lr:1.268e-06, loss:1.188loss:0.2922, grad_norm:0.8607 | |
| 2025-10-26 01:19:24 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28802 | Local Step: 4600 // 12101, total_eta: 83 days, 2:05:16, epoch_eta:12:39:43, time: all:6.087, data:0.001, vae:0.017, IQA:4.261, Connector:0.068, model:1.343, lr:1.273e-06, loss:0.820loss:0.2915, grad_norm:0.4076 | |
| 2025-10-26 01:29:28 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 28902 | Local Step: 4700 // 12101, total_eta: 83 days, 1:37:18, epoch_eta:12:29:29, time: all:6.034, data:0.001, vae:0.018, IQA:4.253, Connector:0.070, model:1.296, lr:1.277e-06, loss:0.461loss:0.2938, grad_norm:0.5410 | |
| 2025-10-26 01:39:53 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29002 | Local Step: 4800 // 12101, total_eta: 83 days, 2:40:46, epoch_eta:12:19:48, time: all:6.256, data:0.001, vae:0.019, IQA:4.356, Connector:0.076, model:1.326, lr:1.281e-06, loss:0.021loss:0.3019, grad_norm:0.3667 | |
| 2025-10-26 01:49:54 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29102 | Local Step: 4900 // 12101, total_eta: 83 days, 2:03:15, epoch_eta:12:09:30, time: all:6.012, data:0.001, vae:0.018, IQA:4.260, Connector:0.069, model:1.270, lr:1.286e-06, loss:0.192loss:0.2954, grad_norm:0.4460 | |
| 2025-10-26 02:00:02 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29202 | Local Step: 5000 // 12101, total_eta: 83 days, 1:53:02, epoch_eta:11:59:22, time: all:6.078, data:0.001, vae:0.018, IQA:4.274, Connector:0.070, model:1.308, lr:1.290e-06, loss:0.104loss:0.2941, grad_norm:0.4096 | |
| 2025-10-26 02:10:02 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29302 | Local Step: 5100 // 12101, total_eta: 83 days, 1:11:50, epoch_eta:11:49:03, time: all:5.998, data:0.001, vae:0.018, IQA:4.223, Connector:0.068, model:1.289, lr:1.295e-06, loss:0.143loss:0.2971, grad_norm:0.4453 | |
| 2025-10-26 02:20:07 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29402 | Local Step: 5200 // 12101, total_eta: 83 days, 0:52:15, epoch_eta:11:38:52, time: all:6.052, data:0.001, vae:0.018, IQA:4.335, Connector:0.067, model:1.228, lr:1.299e-06, loss:0.273loss:0.2782, grad_norm:0.5986 | |
| 2025-10-26 02:30:16 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29502 | Local Step: 5300 // 12101, total_eta: 83 days, 0:47:31, epoch_eta:11:28:47, time: all:6.091, data:0.001, vae:0.018, IQA:4.390, Connector:0.068, model:1.213, lr:1.303e-06, loss:0.449loss:0.2837, grad_norm:0.7132 | |
| 2025-10-26 02:40:22 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29602 | Local Step: 5400 // 12101, total_eta: 83 days, 0:28:56, epoch_eta:11:18:36, time: all:6.053, data:0.001, vae:0.018, IQA:4.345, Connector:0.066, model:1.239, lr:1.308e-06, loss:0.711loss:0.2800, grad_norm:0.4404 | |
| 2025-10-26 02:50:32 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29702 | Local Step: 5500 // 12101, total_eta: 83 days, 0:28:23, epoch_eta:11:08:32, time: all:6.103, data:0.001, vae:0.018, IQA:4.315, Connector:0.069, model:1.299, lr:1.312e-06, loss:0.010loss:0.3097, grad_norm:0.6139 | |
| 2025-10-26 03:00:38 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29802 | Local Step: 5600 // 12101, total_eta: 83 days, 0:14:17, epoch_eta:10:58:23, time: all:6.065, data:0.001, vae:0.018, IQA:4.358, Connector:0.067, model:1.217, lr:1.317e-06, loss:0.014loss:0.2863, grad_norm:118.3945 | |
| 2025-10-26 03:10:43 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 29902 | Local Step: 5700 // 12101, total_eta: 82 days, 23:52:08, epoch_eta:10:48:11, time: all:6.042, data:0.001, vae:0.018, IQA:4.257, Connector:0.070, model:1.292, lr:1.321e-06, loss:0.898loss:0.2959, grad_norm:0.3600 | |
| 2025-10-26 03:20:39 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_30000_weight.pth. | |
| 2025-10-26 03:21:07 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_30000.pth. | |
| 2025-10-26 03:21:43 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30002 | Local Step: 5800 // 12101, total_eta: 83 days, 2:40:09, epoch_eta:10:39:01, time: all:6.601, data:0.001, vae:0.019, IQA:4.387, Connector:0.076, model:1.319, lr:1.326e-06, loss:0.299loss:0.2800, grad_norm:1.2514 | |
| 2025-10-26 03:31:52 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30102 | Local Step: 5900 // 12101, total_eta: 83 days, 2:32:50, epoch_eta:10:28:53, time: all:6.093, data:0.001, vae:0.018, IQA:4.333, Connector:0.067, model:1.277, lr:1.330e-06, loss:0.166loss:0.2826, grad_norm:0.6966 | |
| 2025-10-26 03:41:57 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30202 | Local Step: 6000 // 12101, total_eta: 83 days, 2:12:04, epoch_eta:10:18:41, time: all:6.053, data:0.001, vae:0.018, IQA:4.302, Connector:0.068, model:1.270, lr:1.334e-06, loss:0.426loss:0.2913, grad_norm:0.4671 | |
| 2025-10-26 03:52:00 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30302 | Local Step: 6100 // 12101, total_eta: 83 days, 1:43:33, epoch_eta:10:08:27, time: all:6.028, data:0.001, vae:0.018, IQA:4.246, Connector:0.070, model:1.286, lr:1.339e-06, loss:0.173loss:0.3007, grad_norm:0.5439 | |
| 2025-10-26 04:02:04 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30402 | Local Step: 6200 // 12101, total_eta: 83 days, 1:20:32, epoch_eta:9:58:15, time: all:6.043, data:0.001, vae:0.018, IQA:4.292, Connector:0.068, model:1.259, lr:1.344e-06, loss:0.445loss:0.2764, grad_norm:0.4816 | |
| 2025-10-26 04:12:08 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30502 | Local Step: 6300 // 12101, total_eta: 83 days, 0:55:23, epoch_eta:9:48:02, time: all:6.035, data:0.001, vae:0.018, IQA:4.249, Connector:0.069, model:1.297, lr:1.348e-06, loss:0.301loss:0.2843, grad_norm:0.3675 | |
| 2025-10-26 04:22:09 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30602 | Local Step: 6400 // 12101, total_eta: 83 days, 0:24:39, epoch_eta:9:37:48, time: all:6.015, data:0.001, vae:0.018, IQA:4.221, Connector:0.068, model:1.295, lr:1.352e-06, loss:0.332loss:0.2875, grad_norm:0.6423 | |
| 2025-10-26 04:32:14 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30702 | Local Step: 6500 // 12101, total_eta: 83 days, 0:03:32, epoch_eta:9:27:37, time: all:6.045, data:0.001, vae:0.018, IQA:4.266, Connector:0.068, model:1.280, lr:1.356e-06, loss:0.406loss:0.2855, grad_norm:0.5882 | |
| 2025-10-26 04:42:19 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30802 | Local Step: 6600 // 12101, total_eta: 82 days, 23:43:46, epoch_eta:9:17:26, time: all:6.048, data:0.001, vae:0.018, IQA:4.251, Connector:0.072, model:1.302, lr:1.361e-06, loss:0.410loss:0.2965, grad_norm:0.4139 | |
| 2025-10-26 04:52:21 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 30902 | Local Step: 6700 // 12101, total_eta: 82 days, 23:17:37, epoch_eta:9:07:14, time: all:6.025, data:0.001, vae:0.019, IQA:4.329, Connector:0.069, model:1.226, lr:1.365e-06, loss:0.246loss:0.2922, grad_norm:0.4372 | |
| 2025-10-26 05:02:47 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31002 | Local Step: 6800 // 12101, total_eta: 82 days, 23:57:58, epoch_eta:8:57:19, time: all:6.254, data:0.001, vae:0.019, IQA:4.376, Connector:0.077, model:1.325, lr:1.370e-06, loss:0.209loss:0.2719, grad_norm:0.3304 | |
| 2025-10-26 05:12:52 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31102 | Local Step: 6900 // 12101, total_eta: 82 days, 23:38:32, epoch_eta:8:47:09, time: all:6.049, data:0.001, vae:0.018, IQA:4.288, Connector:0.072, model:1.274, lr:1.374e-06, loss:0.124loss:0.2991, grad_norm:0.4030 | |
| 2025-10-26 05:22:56 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31202 | Local Step: 7000 // 12101, total_eta: 82 days, 23:16:59, epoch_eta:8:36:58, time: all:6.041, data:0.001, vae:0.018, IQA:4.257, Connector:0.068, model:1.313, lr:1.379e-06, loss:0.711loss:0.2954, grad_norm:0.4210 | |
| 2025-10-26 05:32:56 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31302 | Local Step: 7100 // 12101, total_eta: 82 days, 22:46:28, epoch_eta:8:26:44, time: all:6.007, data:0.001, vae:0.018, IQA:4.243, Connector:0.069, model:1.281, lr:1.383e-06, loss:0.106loss:0.2965, grad_norm:0.7051 | |
| 2025-10-26 05:43:03 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31402 | Local Step: 7200 // 12101, total_eta: 82 days, 22:33:18, epoch_eta:8:16:36, time: all:6.069, data:0.001, vae:0.018, IQA:4.284, Connector:0.067, model:1.292, lr:1.387e-06, loss:0.281loss:0.2962, grad_norm:0.4642 | |
| 2025-10-26 05:53:09 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31502 | Local Step: 7300 // 12101, total_eta: 82 days, 22:18:04, epoch_eta:8:06:26, time: all:6.061, data:0.001, vae:0.018, IQA:4.369, Connector:0.069, model:1.204, lr:1.392e-06, loss:0.578loss:0.2811, grad_norm:0.3994 | |
| 2025-10-26 06:03:12 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31602 | Local Step: 7400 // 12101, total_eta: 82 days, 21:53:39, epoch_eta:7:56:15, time: all:6.026, data:0.001, vae:0.018, IQA:4.295, Connector:0.068, model:1.238, lr:1.397e-06, loss:0.371loss:0.2925, grad_norm:1.0647 | |
| 2025-10-26 06:13:17 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31702 | Local Step: 7500 // 12101, total_eta: 82 days, 21:35:32, epoch_eta:7:46:05, time: all:6.048, data:0.001, vae:0.018, IQA:4.268, Connector:0.070, model:1.279, lr:1.401e-06, loss:0.249loss:0.2914, grad_norm:0.4889 | |
| 2025-10-26 06:23:23 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31802 | Local Step: 7600 // 12101, total_eta: 82 days, 21:22:20, epoch_eta:7:35:57, time: all:6.066, data:0.001, vae:0.018, IQA:4.317, Connector:0.068, model:1.265, lr:1.405e-06, loss:0.230loss:0.2836, grad_norm:0.5296 | |
| 2025-10-26 06:33:29 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 31902 | Local Step: 7700 // 12101, total_eta: 82 days, 21:06:50, epoch_eta:7:25:48, time: all:6.057, data:0.001, vae:0.018, IQA:4.273, Connector:0.070, model:1.290, lr:1.409e-06, loss:0.197loss:0.3066, grad_norm:1.0212 | |
| 2025-10-26 06:43:55 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32002 | Local Step: 7800 // 12101, total_eta: 82 days, 21:43:22, epoch_eta:7:15:50, time: all:6.263, data:0.001, vae:0.019, IQA:4.340, Connector:0.075, model:1.379, lr:1.414e-06, loss:0.238loss:0.2746, grad_norm:0.4300 | |
| 2025-10-26 06:54:05 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32102 | Local Step: 7900 // 12101, total_eta: 82 days, 21:38:28, epoch_eta:7:05:43, time: all:6.101, data:0.001, vae:0.017, IQA:4.383, Connector:0.068, model:1.222, lr:1.414e-06, loss:0.551loss:0.2812, grad_norm:0.5441 | |
| 2025-10-26 07:04:13 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32202 | Local Step: 8000 // 12101, total_eta: 82 days, 21:26:09, epoch_eta:6:55:35, time: all:6.071, data:0.001, vae:0.018, IQA:4.264, Connector:0.070, model:1.302, lr:1.414e-06, loss:0.099loss:0.2738, grad_norm:0.4608 | |
| 2025-10-26 07:14:18 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32302 | Local Step: 8100 // 12101, total_eta: 82 days, 21:10:24, epoch_eta:6:45:26, time: all:6.057, data:0.001, vae:0.017, IQA:4.259, Connector:0.070, model:1.299, lr:1.414e-06, loss:0.244loss:0.2952, grad_norm:0.4512 | |
| 2025-10-26 07:24:29 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32402 | Local Step: 8200 // 12101, total_eta: 82 days, 21:07:01, epoch_eta:6:35:19, time: all:6.108, data:0.001, vae:0.018, IQA:4.334, Connector:0.069, model:1.283, lr:1.414e-06, loss:0.178loss:0.2957, grad_norm:0.3210 | |
| 2025-10-26 07:34:31 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32502 | Local Step: 8300 // 12101, total_eta: 82 days, 20:41:24, epoch_eta:6:25:08, time: all:6.015, data:0.001, vae:0.018, IQA:4.333, Connector:0.067, model:1.211, lr:1.414e-06, loss:0.260loss:0.2667, grad_norm:0.8854 | |
| 2025-10-26 07:44:36 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32602 | Local Step: 8400 // 12101, total_eta: 82 days, 20:25:25, epoch_eta:6:14:59, time: all:6.055, data:0.001, vae:0.018, IQA:4.284, Connector:0.068, model:1.283, lr:1.414e-06, loss:0.158loss:0.2941, grad_norm:1.4567 | |
| 2025-10-26 07:54:40 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32702 | Local Step: 8500 // 12101, total_eta: 82 days, 20:07:21, epoch_eta:6:04:50, time: all:6.045, data:0.001, vae:0.018, IQA:4.345, Connector:0.069, model:1.219, lr:1.414e-06, loss:0.656loss:0.2703, grad_norm:0.4603 | |
| 2025-10-26 08:04:48 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32802 | Local Step: 8600 // 12101, total_eta: 82 days, 19:56:20, epoch_eta:5:54:41, time: all:6.075, data:0.001, vae:0.018, IQA:4.346, Connector:0.068, model:1.250, lr:1.414e-06, loss:0.309loss:0.2758, grad_norm:0.3708 | |
| 2025-10-26 08:14:50 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 32902 | Local Step: 8700 // 12101, total_eta: 82 days, 19:33:02, epoch_eta:5:44:31, time: all:6.020, data:0.001, vae:0.018, IQA:4.249, Connector:0.070, model:1.278, lr:1.414e-06, loss:0.166loss:0.2861, grad_norm:0.6947 | |
| 2025-10-26 08:25:12 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33002 | Local Step: 8800 // 12101, total_eta: 82 days, 19:55:23, epoch_eta:5:34:29, time: all:6.224, data:0.001, vae:0.019, IQA:4.336, Connector:0.075, model:1.316, lr:1.414e-06, loss:0.266loss:0.2929, grad_norm:0.7523 | |
| 2025-10-26 08:35:16 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33102 | Local Step: 8900 // 12101, total_eta: 82 days, 19:35:12, epoch_eta:5:24:19, time: all:6.034, data:0.001, vae:0.018, IQA:4.233, Connector:0.072, model:1.301, lr:1.414e-06, loss:0.973loss:0.2900, grad_norm:0.4303 | |
| 2025-10-26 08:45:17 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33202 | Local Step: 9000 // 12101, total_eta: 82 days, 19:10:22, epoch_eta:5:14:09, time: all:6.012, data:0.001, vae:0.018, IQA:4.215, Connector:0.067, model:1.312, lr:1.414e-06, loss:0.106loss:0.2894, grad_norm:0.3809 | |
| 2025-10-26 08:55:25 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33302 | Local Step: 9100 // 12101, total_eta: 82 days, 18:59:53, epoch_eta:5:04:01, time: all:6.077, data:0.001, vae:0.018, IQA:4.315, Connector:0.067, model:1.283, lr:1.414e-06, loss:0.216loss:0.2850, grad_norm:0.3820 | |
| 2025-10-26 09:05:23 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33402 | Local Step: 9200 // 12101, total_eta: 82 days, 18:30:21, epoch_eta:4:53:50, time: all:5.987, data:0.001, vae:0.018, IQA:4.212, Connector:0.071, model:1.286, lr:1.414e-06, loss:0.130loss:0.2960, grad_norm:0.4095 | |
| 2025-10-26 09:15:29 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33502 | Local Step: 9300 // 12101, total_eta: 82 days, 18:16:20, epoch_eta:4:43:42, time: all:6.059, data:0.001, vae:0.018, IQA:4.286, Connector:0.071, model:1.281, lr:1.414e-06, loss:0.707loss:0.2953, grad_norm:1.8395 | |
| 2025-10-26 09:25:37 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33602 | Local Step: 9400 // 12101, total_eta: 82 days, 18:05:59, epoch_eta:4:33:34, time: all:6.076, data:0.001, vae:0.018, IQA:4.262, Connector:0.071, model:1.319, lr:1.414e-06, loss:0.199loss:0.3123, grad_norm:0.4962 | |
| 2025-10-26 09:35:41 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33702 | Local Step: 9500 // 12101, total_eta: 82 days, 17:49:06, epoch_eta:4:23:26, time: all:6.045, data:0.001, vae:0.018, IQA:4.288, Connector:0.070, model:1.262, lr:1.414e-06, loss:0.149loss:0.2776, grad_norm:0.4954 | |
| 2025-10-26 09:45:49 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33802 | Local Step: 9600 // 12101, total_eta: 82 days, 17:38:32, epoch_eta:4:13:18, time: all:6.075, data:0.001, vae:0.018, IQA:4.294, Connector:0.068, model:1.288, lr:1.414e-06, loss:0.145loss:0.2870, grad_norm:0.4647 | |
| 2025-10-26 09:55:55 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 33902 | Local Step: 9700 // 12101, total_eta: 82 days, 17:24:57, epoch_eta:4:03:10, time: all:6.060, data:0.001, vae:0.017, IQA:4.367, Connector:0.069, model:1.213, lr:1.414e-06, loss:0.301loss:0.2808, grad_norm:0.3574 | |
| 2025-10-26 10:06:17 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34002 | Local Step: 9800 // 12101, total_eta: 82 days, 17:42:39, epoch_eta:3:53:05, time: all:6.216, data:0.001, vae:0.019, IQA:4.307, Connector:0.075, model:1.342, lr:1.414e-06, loss:0.262loss:0.2897, grad_norm:0.5678 | |
| 2025-10-26 10:16:23 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34102 | Local Step: 9900 // 12101, total_eta: 82 days, 17:30:25, epoch_eta:3:42:57, time: all:6.068, data:0.001, vae:0.018, IQA:4.345, Connector:0.067, model:1.226, lr:1.414e-06, loss:0.114loss:0.2948, grad_norm:1.0514 | |
| 2025-10-26 10:26:29 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34202 | Local Step: 10000 // 12101, total_eta: 82 days, 17:16:45, epoch_eta:3:32:49, time: all:6.060, data:0.001, vae:0.018, IQA:4.324, Connector:0.069, model:1.235, lr:1.414e-06, loss:0.246loss:0.2788, grad_norm:0.3939 | |
| 2025-10-26 10:36:35 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34302 | Local Step: 10100 // 12101, total_eta: 82 days, 17:02:31, epoch_eta:3:22:41, time: all:6.057, data:0.001, vae:0.017, IQA:4.324, Connector:0.068, model:1.245, lr:1.414e-06, loss:0.348loss:0.2775, grad_norm:0.5101 | |
| 2025-10-26 10:46:43 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34402 | Local Step: 10200 // 12101, total_eta: 82 days, 16:52:04, epoch_eta:3:12:33, time: all:6.076, data:0.001, vae:0.018, IQA:4.355, Connector:0.068, model:1.247, lr:1.414e-06, loss:0.135loss:0.2924, grad_norm:1.7540 | |
| 2025-10-26 10:56:51 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34502 | Local Step: 10300 // 12101, total_eta: 82 days, 16:43:01, epoch_eta:3:02:26, time: all:6.083, data:0.001, vae:0.018, IQA:4.310, Connector:0.070, model:1.287, lr:1.414e-06, loss:0.271loss:0.2848, grad_norm:0.5441 | |
| 2025-10-26 11:06:53 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34602 | Local Step: 10400 // 12101, total_eta: 82 days, 16:22:43, epoch_eta:2:52:17, time: all:6.024, data:0.001, vae:0.018, IQA:4.291, Connector:0.070, model:1.248, lr:1.414e-06, loss:0.012loss:0.2923, grad_norm:0.5671 | |
| 2025-10-26 11:17:00 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34702 | Local Step: 10500 // 12101, total_eta: 82 days, 16:10:11, epoch_eta:2:42:09, time: all:6.064, data:0.001, vae:0.017, IQA:4.284, Connector:0.070, model:1.297, lr:1.414e-06, loss:0.014loss:0.2781, grad_norm:0.5192 | |
| 2025-10-26 11:27:08 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34802 | Local Step: 10600 // 12101, total_eta: 82 days, 16:01:39, epoch_eta:2:32:01, time: all:6.086, data:0.001, vae:0.017, IQA:4.339, Connector:0.068, model:1.247, lr:1.414e-06, loss:0.088loss:0.2589, grad_norm:0.6136 | |
| 2025-10-26 11:37:13 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 34902 | Local Step: 10700 // 12101, total_eta: 82 days, 15:46:05, epoch_eta:2:21:53, time: all:6.047, data:0.001, vae:0.018, IQA:4.278, Connector:0.069, model:1.271, lr:1.414e-06, loss:0.233loss:0.2817, grad_norm:0.4443 | |
| 2025-10-26 11:47:06 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_35000_weight.pth. | |
| 2025-10-26 11:47:33 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_35000.pth. | |
| 2025-10-26 11:48:09 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35002 | Local Step: 10800 // 12101, total_eta: 82 days, 17:03:53, epoch_eta:2:11:51, time: all:6.562, data:0.001, vae:0.019, IQA:4.361, Connector:0.075, model:1.321, lr:1.414e-06, loss:0.268loss:0.2755, grad_norm:0.3245 | |
| 2025-10-26 11:58:16 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35102 | Local Step: 10900 // 12101, total_eta: 82 days, 16:51:10, epoch_eta:2:01:43, time: all:6.067, data:0.001, vae:0.018, IQA:4.337, Connector:0.070, model:1.235, lr:1.414e-06, loss:0.406loss:0.2808, grad_norm:1.3828 | |
| 2025-10-26 12:08:19 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35202 | Local Step: 11000 // 12101, total_eta: 82 days, 16:31:56, epoch_eta:1:51:34, time: all:6.030, data:0.001, vae:0.017, IQA:4.203, Connector:0.069, model:1.327, lr:1.414e-06, loss:0.625loss:0.2822, grad_norm:0.4545 | |
| 2025-10-26 12:18:22 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35302 | Local Step: 11100 // 12101, total_eta: 82 days, 16:12:58, epoch_eta:1:41:26, time: all:6.031, data:0.001, vae:0.017, IQA:4.283, Connector:0.068, model:1.269, lr:1.414e-06, loss:0.523loss:0.2810, grad_norm:0.4948 | |
| 2025-10-26 12:28:24 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35402 | Local Step: 11200 // 12101, total_eta: 82 days, 15:53:02, epoch_eta:1:31:17, time: all:6.024, data:0.001, vae:0.018, IQA:4.211, Connector:0.072, model:1.301, lr:1.414e-06, loss:0.508loss:0.2824, grad_norm:0.6032 | |
| 2025-10-26 12:38:33 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35502 | Local Step: 11300 // 12101, total_eta: 82 days, 15:44:34, epoch_eta:1:21:10, time: all:6.089, data:0.001, vae:0.018, IQA:4.244, Connector:0.072, model:1.347, lr:1.414e-06, loss:0.152loss:0.2773, grad_norm:0.3704 | |
| 2025-10-26 12:48:42 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35602 | Local Step: 11400 // 12101, total_eta: 82 days, 15:35:12, epoch_eta:1:11:02, time: all:6.084, data:0.001, vae:0.018, IQA:4.265, Connector:0.070, model:1.326, lr:1.414e-06, loss:0.008loss:0.2914, grad_norm:0.4325 | |
| 2025-10-26 12:58:52 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35702 | Local Step: 11500 // 12101, total_eta: 82 days, 15:28:40, epoch_eta:1:00:54, time: all:6.101, data:0.001, vae:0.018, IQA:4.374, Connector:0.067, model:1.256, lr:1.414e-06, loss:0.111loss:0.2899, grad_norm:0.4645 | |
| 2025-10-26 13:08:57 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35802 | Local Step: 11600 // 12101, total_eta: 82 days, 15:13:29, epoch_eta:0:50:46, time: all:6.050, data:0.001, vae:0.018, IQA:4.256, Connector:0.070, model:1.315, lr:1.414e-06, loss:0.500loss:0.3018, grad_norm:1.1286 | |
| 2025-10-26 13:19:00 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 35902 | Local Step: 11700 // 12101, total_eta: 82 days, 14:55:44, epoch_eta:0:40:37, time: all:6.034, data:0.001, vae:0.018, IQA:4.263, Connector:0.068, model:1.289, lr:1.414e-06, loss:1.375loss:0.2749, grad_norm:0.6992 | |
| 2025-10-26 13:29:28 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 36002 | Local Step: 11800 // 12101, total_eta: 82 days, 15:17:49, epoch_eta:0:30:30, time: all:6.274, data:0.001, vae:0.020, IQA:4.226, Connector:0.080, model:1.471, lr:1.414e-06, loss:0.508loss:0.3005, grad_norm:1.0974 | |
| 2025-10-26 13:39:35 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 36102 | Local Step: 11900 // 12101, total_eta: 82 days, 15:06:11, epoch_eta:0:20:22, time: all:6.072, data:0.001, vae:0.018, IQA:4.328, Connector:0.070, model:1.247, lr:1.414e-06, loss:0.254loss:0.2951, grad_norm:0.3898 | |
| 2025-10-26 13:49:37 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 36202 | Local Step: 12000 // 12101, total_eta: 82 days, 14:46:27, epoch_eta:0:10:14, time: all:6.022, data:0.001, vae:0.018, IQA:4.347, Connector:0.068, model:1.202, lr:1.414e-06, loss:0.010loss:0.2757, grad_norm:0.4847 | |
| 2025-10-26 13:59:41 - [1m[Sana][0m - INFO - Epoch: 3 | Global Step: 36302 | Local Step: 12100 // 12101, total_eta: 82 days, 14:29:54, epoch_eta:0:00:06, time: all:6.041, data:0.001, vae:0.017, IQA:4.205, Connector:0.067, model:1.345, lr:1.414e-06, loss:0.220loss:0.2911, grad_norm:0.4183 | |
| 2025-10-26 13:59:53 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_36304_weight.pth. | |
| 2025-10-26 14:00:22 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 3 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_3_step_36304.pth. | |
| 2025-10-26 14:00:31 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 36304 | Local Step: 1 // 12101, total_eta: 117 days, 18:13:00, epoch_eta:1 day, 5:08:02, time: all:0.087, data:0.016, vae:0.000, IQA:0.049, Connector:0.001, model:0.014, lr:1.414e-06, loss:0.344loss:0.3174, grad_norm:0.8398 | |
| 2025-10-26 14:10:45 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 36403 | Local Step: 100 // 12101, total_eta: 84 days, 13:26:35, epoch_eta:20:45:03, time: all:6.138, data:0.001, vae:0.020, IQA:4.227, Connector:0.074, model:1.351, lr:1.414e-06, loss:0.018loss:0.2779, grad_norm:0.3438 | |
| 2025-10-26 14:20:47 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 36503 | Local Step: 200 // 12101, total_eta: 83 days, 5:08:29, epoch_eta:20:15:08, time: all:6.028, data:0.001, vae:0.018, IQA:4.286, Connector:0.069, model:1.250, lr:1.414e-06, loss:0.014loss:0.3050, grad_norm:0.5037 | |
| 2025-10-26 14:30:54 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 36603 | Local Step: 300 // 12101, total_eta: 82 days, 22:54:56, epoch_eta:20:01:16, time: all:6.070, data:0.001, vae:0.018, IQA:4.270, Connector:0.070, model:1.301, lr:1.414e-06, loss:0.099loss:0.2850, grad_norm:0.5410 | |
| 2025-10-26 14:40:59 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 36703 | Local Step: 400 // 12101, total_eta: 82 days, 17:58:17, epoch_eta:19:48:14, time: all:6.049, data:0.001, vae:0.018, IQA:4.351, Connector:0.067, model:1.213, lr:1.414e-06, loss:0.130loss:0.2778, grad_norm:0.6540 | |
| 2025-10-26 14:51:03 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 36803 | Local Step: 500 // 12101, total_eta: 82 days, 14:15:36, epoch_eta:19:35:58, time: all:6.039, data:0.001, vae:0.018, IQA:4.250, Connector:0.070, model:1.293, lr:1.414e-06, loss:0.297loss:0.2901, grad_norm:0.5672 | |
| 2025-10-26 15:01:06 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 36903 | Local Step: 600 // 12101, total_eta: 82 days, 10:53:38, epoch_eta:19:23:57, time: all:6.023, data:0.001, vae:0.018, IQA:4.342, Connector:0.067, model:1.207, lr:1.414e-06, loss:0.289loss:0.2662, grad_norm:0.6012 | |
| 2025-10-26 15:11:30 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37003 | Local Step: 700 // 12101, total_eta: 82 days, 18:36:27, epoch_eta:19:18:26, time: all:6.242, data:0.001, vae:0.019, IQA:4.362, Connector:0.074, model:1.317, lr:1.414e-06, loss:0.270loss:0.2833, grad_norm:0.4697 | |
| 2025-10-26 15:21:30 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37103 | Local Step: 800 // 12101, total_eta: 82 days, 14:46:03, epoch_eta:19:06:09, time: all:6.006, data:0.001, vae:0.019, IQA:4.234, Connector:0.071, model:1.283, lr:1.414e-06, loss:0.018loss:0.2855, grad_norm:0.5072 | |
| 2025-10-26 15:31:37 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37203 | Local Step: 900 // 12101, total_eta: 82 days, 13:58:52, epoch_eta:18:55:39, time: all:6.068, data:0.001, vae:0.018, IQA:4.375, Connector:0.067, model:1.206, lr:1.414e-06, loss:0.348loss:0.2878, grad_norm:0.7483 | |
| 2025-10-26 15:41:43 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37303 | Local Step: 1000 // 12101, total_eta: 82 days, 13:06:16, epoch_eta:18:45:07, time: all:6.062, data:0.001, vae:0.018, IQA:4.322, Connector:0.068, model:1.250, lr:1.414e-06, loss:1.031loss:0.2626, grad_norm:0.3574 | |
| 2025-10-26 15:51:46 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37403 | Local Step: 1100 // 12101, total_eta: 82 days, 11:16:08, epoch_eta:18:34:02, time: all:6.025, data:0.001, vae:0.018, IQA:4.318, Connector:0.068, model:1.230, lr:1.414e-06, loss:0.212loss:0.2773, grad_norm:0.3691 | |
| 2025-10-26 16:01:52 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37503 | Local Step: 1200 // 12101, total_eta: 82 days, 10:36:53, epoch_eta:18:23:38, time: all:6.058, data:0.001, vae:0.018, IQA:4.299, Connector:0.070, model:1.261, lr:1.414e-06, loss:0.867loss:0.2962, grad_norm:0.4792 | |
| 2025-10-26 16:12:00 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37603 | Local Step: 1300 // 12101, total_eta: 82 days, 10:38:37, epoch_eta:18:13:37, time: all:6.082, data:0.001, vae:0.018, IQA:4.276, Connector:0.070, model:1.307, lr:1.414e-06, loss:0.070loss:0.2797, grad_norm:0.5498 | |
| 2025-10-26 16:22:08 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37703 | Local Step: 1400 // 12101, total_eta: 82 days, 10:43:20, epoch_eta:18:03:38, time: all:6.086, data:0.001, vae:0.018, IQA:4.333, Connector:0.069, model:1.264, lr:1.414e-06, loss:0.295loss:0.2746, grad_norm:0.3796 | |
| 2025-10-26 16:32:18 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37803 | Local Step: 1500 // 12101, total_eta: 82 days, 10:53:18, epoch_eta:17:53:41, time: all:6.091, data:0.001, vae:0.018, IQA:4.332, Connector:0.070, model:1.262, lr:1.414e-06, loss:0.010loss:0.2880, grad_norm:0.3706 | |
| 2025-10-26 16:42:23 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 37903 | Local Step: 1600 // 12101, total_eta: 82 days, 10:13:46, epoch_eta:17:43:18, time: all:6.053, data:0.001, vae:0.018, IQA:4.303, Connector:0.071, model:1.253, lr:1.414e-06, loss:0.099loss:0.2783, grad_norm:0.3812 | |
| 2025-10-26 16:52:43 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38003 | Local Step: 1700 // 12101, total_eta: 82 days, 12:23:30, epoch_eta:17:34:25, time: all:6.197, data:0.001, vae:0.019, IQA:4.355, Connector:0.079, model:1.280, lr:1.414e-06, loss:0.387loss:0.2785, grad_norm:0.7088 | |
| 2025-10-26 17:02:44 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38103 | Local Step: 1800 // 12101, total_eta: 82 days, 11:03:32, epoch_eta:17:23:40, time: all:6.018, data:0.001, vae:0.018, IQA:4.293, Connector:0.069, model:1.236, lr:1.414e-06, loss:0.328loss:0.2865, grad_norm:0.5454 | |
| 2025-10-26 17:12:49 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38203 | Local Step: 1900 // 12101, total_eta: 82 days, 10:17:50, epoch_eta:17:13:13, time: all:6.044, data:0.001, vae:0.018, IQA:4.323, Connector:0.070, model:1.252, lr:1.414e-06, loss:0.539loss:0.2651, grad_norm:0.5889 | |
| 2025-10-26 17:22:49 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38303 | Local Step: 2000 // 12101, total_eta: 82 days, 8:53:26, epoch_eta:17:02:27, time: all:6.001, data:0.001, vae:0.018, IQA:4.245, Connector:0.069, model:1.282, lr:1.414e-06, loss:0.730loss:0.2916, grad_norm:0.4604 | |
| 2025-10-26 17:32:53 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38403 | Local Step: 2100 // 12101, total_eta: 82 days, 8:10:21, epoch_eta:16:52:03, time: all:6.038, data:0.001, vae:0.018, IQA:4.317, Connector:0.067, model:1.266, lr:1.414e-06, loss:0.105loss:0.2893, grad_norm:0.3803 | |
| 2025-10-26 17:42:55 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38503 | Local Step: 2200 // 12101, total_eta: 82 days, 7:16:49, epoch_eta:16:41:34, time: all:6.023, data:0.001, vae:0.018, IQA:4.333, Connector:0.065, model:1.235, lr:1.414e-06, loss:0.574loss:0.2785, grad_norm:0.5856 | |
| 2025-10-26 17:52:56 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38603 | Local Step: 2300 // 12101, total_eta: 82 days, 6:18:02, epoch_eta:16:31:02, time: all:6.012, data:0.001, vae:0.018, IQA:4.270, Connector:0.068, model:1.272, lr:1.414e-06, loss:0.373loss:0.2868, grad_norm:0.5078 | |
| 2025-10-26 18:02:58 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38703 | Local Step: 2400 // 12101, total_eta: 82 days, 5:28:01, epoch_eta:16:20:36, time: all:6.018, data:0.001, vae:0.018, IQA:4.300, Connector:0.069, model:1.249, lr:1.414e-06, loss:0.285loss:0.2644, grad_norm:0.6836 | |
| 2025-10-26 18:13:01 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38803 | Local Step: 2500 // 12101, total_eta: 82 days, 4:46:09, epoch_eta:16:10:14, time: all:6.024, data:0.001, vae:0.018, IQA:4.279, Connector:0.067, model:1.280, lr:1.414e-06, loss:0.156loss:0.2714, grad_norm:0.3929 | |
| 2025-10-26 18:23:06 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 38903 | Local Step: 2600 // 12101, total_eta: 82 days, 4:25:59, epoch_eta:16:00:02, time: all:6.050, data:0.001, vae:0.018, IQA:4.401, Connector:0.066, model:1.196, lr:1.414e-06, loss:0.633loss:0.2883, grad_norm:0.4391 | |
| 2025-10-26 18:33:26 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39003 | Local Step: 2700 // 12101, total_eta: 82 days, 5:55:35, epoch_eta:15:50:44, time: all:6.201, data:0.001, vae:0.019, IQA:4.339, Connector:0.076, model:1.301, lr:1.414e-06, loss:1.000loss:0.2762, grad_norm:1.4492 | |
| 2025-10-26 18:43:29 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39103 | Local Step: 2800 // 12101, total_eta: 82 days, 5:19:11, epoch_eta:15:40:25, time: all:6.030, data:0.001, vae:0.018, IQA:4.345, Connector:0.069, model:1.223, lr:1.414e-06, loss:0.111loss:0.2807, grad_norm:1.7422 | |
| 2025-10-26 18:53:36 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39203 | Local Step: 2900 // 12101, total_eta: 82 days, 5:11:55, epoch_eta:15:30:20, time: all:6.071, data:0.001, vae:0.018, IQA:4.344, Connector:0.069, model:1.252, lr:1.414e-06, loss:0.256loss:0.2817, grad_norm:0.3691 | |
| 2025-10-26 19:03:43 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39303 | Local Step: 3000 // 12101, total_eta: 82 days, 5:07:46, epoch_eta:15:20:16, time: all:6.076, data:0.001, vae:0.018, IQA:4.346, Connector:0.069, model:1.265, lr:1.414e-06, loss:0.148loss:0.2791, grad_norm:0.5094 | |
| 2025-10-26 19:13:48 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39403 | Local Step: 3100 // 12101, total_eta: 82 days, 4:43:38, epoch_eta:15:10:02, time: all:6.045, data:0.001, vae:0.018, IQA:4.299, Connector:0.068, model:1.285, lr:1.414e-06, loss:0.068loss:0.2787, grad_norm:0.5449 | |
| 2025-10-26 19:23:50 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39503 | Local Step: 3200 // 12101, total_eta: 82 days, 4:05:17, epoch_eta:14:59:43, time: all:6.020, data:0.001, vae:0.019, IQA:4.280, Connector:0.070, model:1.275, lr:1.414e-06, loss:0.543loss:0.2695, grad_norm:0.4644 | |
| 2025-10-26 19:33:59 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39603 | Local Step: 3300 // 12101, total_eta: 82 days, 4:09:47, epoch_eta:14:49:43, time: all:6.090, data:0.001, vae:0.019, IQA:4.318, Connector:0.069, model:1.297, lr:1.414e-06, loss:0.097loss:0.2866, grad_norm:0.5187 | |
| 2025-10-26 19:44:02 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39703 | Local Step: 3400 // 12101, total_eta: 82 days, 3:39:32, epoch_eta:14:39:28, time: all:6.031, data:0.001, vae:0.018, IQA:4.281, Connector:0.068, model:1.284, lr:1.414e-06, loss:0.129loss:0.2798, grad_norm:0.3151 | |
| 2025-10-26 19:54:06 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39803 | Local Step: 3500 // 12101, total_eta: 82 days, 3:17:55, epoch_eta:14:29:16, time: all:6.044, data:0.001, vae:0.018, IQA:4.298, Connector:0.070, model:1.281, lr:1.414e-06, loss:0.127loss:0.2865, grad_norm:0.3830 | |
| 2025-10-26 20:04:11 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 39903 | Local Step: 3600 // 12101, total_eta: 82 days, 3:01:29, epoch_eta:14:19:07, time: all:6.052, data:0.001, vae:0.018, IQA:4.391, Connector:0.069, model:1.207, lr:1.414e-06, loss:0.019loss:0.2753, grad_norm:0.7246 | |
| 2025-10-26 20:13:58 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 4 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_4_step_40000_weight.pth. | |
| 2025-10-26 20:14:27 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 4 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_4_step_40000.pth. | |
| 2025-10-26 20:15:08 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40003 | Local Step: 3700 // 12101, total_eta: 82 days, 7:17:25, epoch_eta:14:10:55, time: all:6.568, data:0.001, vae:0.019, IQA:4.292, Connector:0.074, model:1.398, lr:1.414e-06, loss:0.014loss:0.2905, grad_norm:0.8378 | |
| 2025-10-26 20:25:14 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40103 | Local Step: 3800 // 12101, total_eta: 82 days, 6:56:00, epoch_eta:14:00:43, time: all:6.055, data:0.001, vae:0.018, IQA:4.271, Connector:0.070, model:1.305, lr:1.414e-06, loss:0.352loss:0.2840, grad_norm:0.5563 | |
| 2025-10-26 20:35:15 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40203 | Local Step: 3900 // 12101, total_eta: 82 days, 6:14:33, epoch_eta:13:50:22, time: all:6.014, data:0.001, vae:0.019, IQA:4.231, Connector:0.070, model:1.311, lr:1.414e-06, loss:0.178loss:0.2745, grad_norm:0.3939 | |
| 2025-10-26 20:45:20 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40303 | Local Step: 4000 // 12101, total_eta: 82 days, 5:53:12, epoch_eta:13:40:09, time: all:6.052, data:0.001, vae:0.018, IQA:4.331, Connector:0.070, model:1.256, lr:1.414e-06, loss:0.355loss:0.2835, grad_norm:0.4694 | |
| 2025-10-26 20:55:21 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40403 | Local Step: 4100 // 12101, total_eta: 82 days, 5:10:10, epoch_eta:13:29:49, time: all:6.005, data:0.001, vae:0.018, IQA:4.241, Connector:0.070, model:1.293, lr:1.414e-06, loss:0.104loss:0.2851, grad_norm:0.5359 | |
| 2025-10-26 21:05:24 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40503 | Local Step: 4200 // 12101, total_eta: 82 days, 4:38:57, epoch_eta:13:19:33, time: all:6.027, data:0.001, vae:0.018, IQA:4.347, Connector:0.070, model:1.217, lr:1.414e-06, loss:0.007loss:0.2878, grad_norm:0.4595 | |
| 2025-10-26 21:15:28 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40603 | Local Step: 4300 // 12101, total_eta: 82 days, 4:16:00, epoch_eta:13:09:20, time: all:6.044, data:0.001, vae:0.018, IQA:4.310, Connector:0.068, model:1.266, lr:1.414e-06, loss:0.189loss:0.2638, grad_norm:0.3978 | |
| 2025-10-26 21:25:34 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40703 | Local Step: 4400 // 12101, total_eta: 82 days, 3:59:44, epoch_eta:12:59:11, time: all:6.057, data:0.001, vae:0.018, IQA:4.349, Connector:0.067, model:1.240, lr:1.414e-06, loss:0.195loss:0.2976, grad_norm:0.4375 | |
| 2025-10-26 21:35:41 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40803 | Local Step: 4500 // 12101, total_eta: 82 days, 3:49:08, epoch_eta:12:49:04, time: all:6.070, data:0.001, vae:0.018, IQA:4.352, Connector:0.070, model:1.242, lr:1.414e-06, loss:0.215loss:0.2731, grad_norm:0.4718 | |
| 2025-10-26 21:45:43 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 40903 | Local Step: 4600 // 12101, total_eta: 82 days, 3:20:46, epoch_eta:12:38:49, time: all:6.028, data:0.001, vae:0.018, IQA:4.288, Connector:0.070, model:1.264, lr:1.414e-06, loss:0.182loss:0.2914, grad_norm:0.6081 | |
| 2025-10-26 21:56:06 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41003 | Local Step: 4700 // 12101, total_eta: 82 days, 4:15:01, epoch_eta:12:29:07, time: all:6.225, data:0.001, vae:0.019, IQA:4.379, Connector:0.077, model:1.297, lr:1.414e-06, loss:0.330loss:0.2707, grad_norm:0.4333 | |
| 2025-10-26 22:06:07 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41103 | Local Step: 4800 // 12101, total_eta: 82 days, 3:40:34, epoch_eta:12:18:50, time: all:6.013, data:0.001, vae:0.019, IQA:4.304, Connector:0.066, model:1.240, lr:1.414e-06, loss:0.012loss:0.2867, grad_norm:0.7406 | |
| 2025-10-26 22:16:17 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41203 | Local Step: 4900 // 12101, total_eta: 82 days, 3:39:43, epoch_eta:12:08:47, time: all:6.095, data:0.001, vae:0.019, IQA:4.391, Connector:0.069, model:1.233, lr:1.414e-06, loss:0.145loss:0.2962, grad_norm:0.5243 | |
| 2025-10-26 22:26:23 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41303 | Local Step: 5000 // 12101, total_eta: 82 days, 3:26:00, epoch_eta:11:58:38, time: all:6.063, data:0.001, vae:0.018, IQA:4.377, Connector:0.067, model:1.234, lr:1.414e-06, loss:0.032loss:0.2658, grad_norm:0.4079 | |
| 2025-10-26 22:36:20 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41403 | Local Step: 5100 // 12101, total_eta: 82 days, 2:36:14, epoch_eta:11:48:17, time: all:5.968, data:0.001, vae:0.018, IQA:4.197, Connector:0.070, model:1.292, lr:1.414e-06, loss:0.188loss:0.2845, grad_norm:0.4027 | |
| 2025-10-26 22:46:25 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41503 | Local Step: 5200 // 12101, total_eta: 82 days, 2:18:58, epoch_eta:11:38:07, time: all:6.051, data:0.001, vae:0.018, IQA:4.267, Connector:0.068, model:1.307, lr:1.414e-06, loss:0.097loss:0.2716, grad_norm:0.4481 | |
| 2025-10-26 22:56:30 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41603 | Local Step: 5300 // 12101, total_eta: 82 days, 2:01:34, epoch_eta:11:27:58, time: all:6.050, data:0.001, vae:0.018, IQA:4.267, Connector:0.070, model:1.316, lr:1.414e-06, loss:0.204loss:0.2710, grad_norm:0.4392 | |
| 2025-10-26 23:06:34 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41703 | Local Step: 5400 // 12101, total_eta: 82 days, 1:40:49, epoch_eta:11:17:47, time: all:6.040, data:0.001, vae:0.018, IQA:4.301, Connector:0.069, model:1.256, lr:1.414e-06, loss:0.020loss:0.2864, grad_norm:0.6488 | |
| 2025-10-26 23:16:39 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41803 | Local Step: 5500 // 12101, total_eta: 82 days, 1:23:08, epoch_eta:11:07:38, time: all:6.048, data:0.001, vae:0.018, IQA:4.323, Connector:0.068, model:1.259, lr:1.414e-06, loss:0.381loss:0.2749, grad_norm:0.6751 | |
| 2025-10-26 23:26:47 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 41903 | Local Step: 5600 // 12101, total_eta: 82 days, 1:18:26, epoch_eta:10:57:33, time: all:6.084, data:0.001, vae:0.018, IQA:4.286, Connector:0.070, model:1.321, lr:1.414e-06, loss:0.114loss:0.2805, grad_norm:0.5651 | |
| 2025-10-26 23:37:12 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42003 | Local Step: 5700 // 12101, total_eta: 82 days, 2:08:03, epoch_eta:10:47:45, time: all:6.244, data:0.001, vae:0.020, IQA:4.335, Connector:0.077, model:1.370, lr:1.414e-06, loss:0.277loss:0.2754, grad_norm:2.3025 | |
| 2025-10-26 23:47:11 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42103 | Local Step: 5800 // 12101, total_eta: 82 days, 1:31:10, epoch_eta:10:37:29, time: all:5.992, data:0.001, vae:0.018, IQA:4.240, Connector:0.069, model:1.288, lr:1.414e-06, loss:0.182loss:0.2727, grad_norm:0.4967 | |
| 2025-10-26 23:57:13 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42203 | Local Step: 5900 // 12101, total_eta: 82 days, 1:04:57, epoch_eta:10:27:17, time: all:6.022, data:0.001, vae:0.018, IQA:4.316, Connector:0.068, model:1.249, lr:1.414e-06, loss:0.160loss:0.2672, grad_norm:0.3517 | |
| 2025-10-27 00:07:11 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42303 | Local Step: 6000 // 12101, total_eta: 82 days, 0:25:59, epoch_eta:10:17:01, time: all:5.981, data:0.001, vae:0.018, IQA:4.288, Connector:0.070, model:1.240, lr:1.414e-06, loss:0.426loss:0.2829, grad_norm:0.3682 | |
| 2025-10-27 00:17:18 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42403 | Local Step: 6100 // 12101, total_eta: 82 days, 0:14:44, epoch_eta:10:06:54, time: all:6.065, data:0.001, vae:0.019, IQA:4.251, Connector:0.070, model:1.329, lr:1.414e-06, loss:0.186loss:0.2716, grad_norm:0.4393 | |
| 2025-10-27 00:27:24 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42503 | Local Step: 6200 // 12101, total_eta: 82 days, 0:05:01, epoch_eta:9:56:47, time: all:6.069, data:0.001, vae:0.018, IQA:4.363, Connector:0.069, model:1.241, lr:1.414e-06, loss:0.128loss:0.2609, grad_norm:0.6686 | |
| 2025-10-27 00:37:34 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42603 | Local Step: 6300 // 12101, total_eta: 82 days, 0:03:59, epoch_eta:9:46:43, time: all:6.097, data:0.001, vae:0.018, IQA:4.331, Connector:0.069, model:1.277, lr:1.414e-06, loss:0.007loss:0.2720, grad_norm:0.3317 | |
| 2025-10-27 00:47:41 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42703 | Local Step: 6400 // 12101, total_eta: 81 days, 23:53:13, epoch_eta:9:36:36, time: all:6.066, data:0.001, vae:0.018, IQA:4.345, Connector:0.068, model:1.244, lr:1.414e-06, loss:0.100loss:0.2706, grad_norm:0.5160 | |
| 2025-10-27 00:57:47 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42803 | Local Step: 6500 // 12101, total_eta: 81 days, 23:41:58, epoch_eta:9:26:29, time: all:6.065, data:0.001, vae:0.018, IQA:4.331, Connector:0.068, model:1.252, lr:1.414e-06, loss:0.383loss:0.2576, grad_norm:0.3694 | |
| 2025-10-27 01:07:52 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 42903 | Local Step: 6600 // 12101, total_eta: 81 days, 23:25:51, epoch_eta:9:16:21, time: all:6.048, data:0.001, vae:0.018, IQA:4.324, Connector:0.070, model:1.251, lr:1.414e-06, loss:0.256loss:0.2775, grad_norm:0.4528 | |
| 2025-10-27 01:18:15 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43003 | Local Step: 6700 // 12101, total_eta: 82 days, 0:01:02, epoch_eta:9:06:26, time: all:6.224, data:0.001, vae:0.019, IQA:4.290, Connector:0.077, model:1.361, lr:1.414e-06, loss:0.262loss:0.2747, grad_norm:0.4924 | |
| 2025-10-27 01:28:21 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43103 | Local Step: 6800 // 12101, total_eta: 81 days, 23:48:46, epoch_eta:8:56:19, time: all:6.063, data:0.001, vae:0.018, IQA:4.292, Connector:0.069, model:1.285, lr:1.414e-06, loss:0.085loss:0.2864, grad_norm:0.6895 | |
| 2025-10-27 01:38:26 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43203 | Local Step: 6900 // 12101, total_eta: 81 days, 23:32:47, epoch_eta:8:46:10, time: all:6.050, data:0.001, vae:0.018, IQA:4.336, Connector:0.073, model:1.217, lr:1.414e-06, loss:0.992loss:0.2754, grad_norm:0.4436 | |
| 2025-10-27 01:48:30 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43303 | Local Step: 7000 // 12101, total_eta: 81 days, 23:15:11, epoch_eta:8:36:01, time: all:6.043, data:0.001, vae:0.018, IQA:4.283, Connector:0.067, model:1.271, lr:1.414e-06, loss:0.273loss:0.2892, grad_norm:0.6465 | |
| 2025-10-27 01:58:36 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43403 | Local Step: 7100 // 12101, total_eta: 81 days, 23:00:47, epoch_eta:8:25:53, time: all:6.054, data:0.001, vae:0.017, IQA:4.267, Connector:0.068, model:1.296, lr:1.414e-06, loss:0.184loss:0.2656, grad_norm:0.8527 | |
| 2025-10-27 02:08:41 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43503 | Local Step: 7200 // 12101, total_eta: 81 days, 22:47:32, epoch_eta:8:15:45, time: all:6.058, data:0.001, vae:0.018, IQA:4.328, Connector:0.067, model:1.242, lr:1.414e-06, loss:0.133loss:0.2770, grad_norm:0.3809 | |
| 2025-10-27 02:18:47 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43603 | Local Step: 7300 // 12101, total_eta: 81 days, 22:33:18, epoch_eta:8:05:37, time: all:6.054, data:0.001, vae:0.018, IQA:4.284, Connector:0.069, model:1.274, lr:1.414e-06, loss:1.000loss:0.2654, grad_norm:0.4668 | |
| 2025-10-27 02:28:44 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43703 | Local Step: 7400 // 12101, total_eta: 81 days, 21:57:57, epoch_eta:7:55:24, time: all:5.973, data:0.001, vae:0.018, IQA:4.227, Connector:0.069, model:1.259, lr:1.414e-06, loss:0.013loss:0.2810, grad_norm:1.3050 | |
| 2025-10-27 02:38:46 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43803 | Local Step: 7500 // 12101, total_eta: 81 days, 21:36:20, epoch_eta:7:45:15, time: all:6.023, data:0.001, vae:0.018, IQA:4.333, Connector:0.069, model:1.209, lr:1.414e-06, loss:0.443loss:0.2856, grad_norm:0.6064 | |
| 2025-10-27 02:48:51 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 43903 | Local Step: 7600 // 12101, total_eta: 81 days, 21:21:28, epoch_eta:7:35:07, time: all:6.049, data:0.001, vae:0.018, IQA:4.254, Connector:0.068, model:1.320, lr:1.414e-06, loss:0.609loss:0.2737, grad_norm:0.4945 | |
| 2025-10-27 02:59:16 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44003 | Local Step: 7700 // 12101, total_eta: 81 days, 21:55:45, epoch_eta:7:25:10, time: all:6.243, data:0.001, vae:0.019, IQA:4.341, Connector:0.078, model:1.348, lr:1.414e-06, loss:0.379loss:0.2831, grad_norm:0.3806 | |
| 2025-10-27 03:09:19 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44103 | Local Step: 7800 // 12101, total_eta: 81 days, 21:38:02, epoch_eta:7:15:02, time: all:6.039, data:0.001, vae:0.018, IQA:4.197, Connector:0.071, model:1.357, lr:1.414e-06, loss:0.418loss:0.2652, grad_norm:0.4596 | |
| 2025-10-27 03:19:26 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44203 | Local Step: 7900 // 12101, total_eta: 81 days, 21:26:52, epoch_eta:7:04:55, time: all:6.065, data:0.001, vae:0.018, IQA:4.353, Connector:0.068, model:1.255, lr:1.414e-06, loss:0.141loss:0.2661, grad_norm:0.4897 | |
| 2025-10-27 03:29:25 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44303 | Local Step: 8000 // 12101, total_eta: 81 days, 20:57:27, epoch_eta:6:54:44, time: all:5.989, data:0.001, vae:0.018, IQA:4.259, Connector:0.070, model:1.253, lr:1.414e-06, loss:0.097loss:0.2830, grad_norm:0.7026 | |
| 2025-10-27 03:39:28 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44403 | Local Step: 8100 // 12101, total_eta: 81 days, 20:38:14, epoch_eta:6:44:35, time: all:6.030, data:0.001, vae:0.018, IQA:4.249, Connector:0.067, model:1.312, lr:1.414e-06, loss:0.127loss:0.2540, grad_norm:0.3518 | |
| 2025-10-27 03:49:25 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44503 | Local Step: 8200 // 12101, total_eta: 81 days, 20:06:43, epoch_eta:6:34:24, time: all:5.977, data:0.001, vae:0.018, IQA:4.206, Connector:0.068, model:1.302, lr:1.414e-06, loss:0.139loss:0.2731, grad_norm:0.4482 | |
| 2025-10-27 03:59:25 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44603 | Local Step: 8300 // 12101, total_eta: 81 days, 19:38:52, epoch_eta:6:24:14, time: all:5.990, data:0.001, vae:0.018, IQA:4.305, Connector:0.068, model:1.226, lr:1.414e-06, loss:0.314loss:0.2698, grad_norm:0.3779 | |
| 2025-10-27 04:09:35 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44703 | Local Step: 8400 // 12101, total_eta: 81 days, 19:37:02, epoch_eta:6:14:09, time: all:6.101, data:0.001, vae:0.018, IQA:4.344, Connector:0.068, model:1.284, lr:1.414e-06, loss:0.430loss:0.2768, grad_norm:0.5669 | |
| 2025-10-27 04:19:39 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44803 | Local Step: 8500 // 12101, total_eta: 81 days, 19:21:05, epoch_eta:6:04:01, time: all:6.040, data:0.001, vae:0.019, IQA:4.287, Connector:0.069, model:1.273, lr:1.414e-06, loss:0.234loss:0.2810, grad_norm:1.0385 | |
| 2025-10-27 04:29:46 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 44903 | Local Step: 8600 // 12101, total_eta: 81 days, 19:12:29, epoch_eta:5:53:55, time: all:6.072, data:0.001, vae:0.018, IQA:4.371, Connector:0.069, model:1.229, lr:1.414e-06, loss:0.124loss:0.2743, grad_norm:0.4749 | |
| 2025-10-27 04:39:38 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 4 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_4_step_45000_weight.pth. | |
| 2025-10-27 04:40:11 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 4 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_4_step_45000.pth. | |
| 2025-10-27 04:40:53 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45003 | Local Step: 8700 // 12101, total_eta: 81 days, 21:17:39, epoch_eta:5:44:12, time: all:6.672, data:0.001, vae:0.019, IQA:4.384, Connector:0.074, model:1.307, lr:1.414e-06, loss:0.104loss:0.2721, grad_norm:0.6787 | |
| 2025-10-27 04:50:59 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45103 | Local Step: 8800 // 12101, total_eta: 81 days, 21:03:43, epoch_eta:5:34:04, time: all:6.055, data:0.001, vae:0.018, IQA:4.313, Connector:0.069, model:1.274, lr:1.414e-06, loss:0.012loss:0.2681, grad_norm:0.6676 | |
| 2025-10-27 05:01:02 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45203 | Local Step: 8900 // 12101, total_eta: 81 days, 20:44:50, epoch_eta:5:23:56, time: all:6.032, data:0.001, vae:0.018, IQA:4.201, Connector:0.071, model:1.359, lr:1.414e-06, loss:0.324loss:0.2723, grad_norm:0.5378 | |
| 2025-10-27 05:11:06 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45303 | Local Step: 9000 // 12101, total_eta: 81 days, 20:27:33, epoch_eta:5:13:47, time: all:6.039, data:0.001, vae:0.018, IQA:4.262, Connector:0.068, model:1.306, lr:1.414e-06, loss:0.262loss:0.2851, grad_norm:0.5412 | |
| 2025-10-27 05:21:03 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45403 | Local Step: 9100 // 12101, total_eta: 81 days, 19:57:39, epoch_eta:5:03:37, time: all:5.979, data:0.001, vae:0.019, IQA:4.230, Connector:0.071, model:1.279, lr:1.414e-06, loss:0.012loss:0.2697, grad_norm:0.5542 | |
| 2025-10-27 05:31:10 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45503 | Local Step: 9200 // 12101, total_eta: 81 days, 19:45:45, epoch_eta:4:53:30, time: all:6.062, data:0.001, vae:0.018, IQA:4.247, Connector:0.068, model:1.351, lr:1.414e-06, loss:0.164loss:0.2853, grad_norm:0.3776 | |
| 2025-10-27 05:41:16 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45603 | Local Step: 9300 // 12101, total_eta: 81 days, 19:33:28, epoch_eta:4:43:22, time: all:6.060, data:0.001, vae:0.018, IQA:4.277, Connector:0.071, model:1.303, lr:1.414e-06, loss:0.305loss:0.2723, grad_norm:0.5103 | |
| 2025-10-27 05:51:24 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45703 | Local Step: 9400 // 12101, total_eta: 81 days, 19:25:10, epoch_eta:4:33:16, time: all:6.079, data:0.001, vae:0.019, IQA:4.430, Connector:0.064, model:1.198, lr:1.414e-06, loss:0.112loss:0.2711, grad_norm:0.3050 | |
| 2025-10-27 06:01:22 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45803 | Local Step: 9500 // 12101, total_eta: 81 days, 18:57:22, epoch_eta:4:23:06, time: all:5.984, data:0.001, vae:0.018, IQA:4.264, Connector:0.070, model:1.249, lr:1.414e-06, loss:0.410loss:0.2804, grad_norm:0.5879 | |
| 2025-10-27 06:11:24 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 45903 | Local Step: 9600 // 12101, total_eta: 81 days, 18:37:23, epoch_eta:4:12:58, time: all:6.021, data:0.001, vae:0.018, IQA:4.305, Connector:0.068, model:1.260, lr:1.414e-06, loss:0.114loss:0.2867, grad_norm:0.3761 | |
| 2025-10-27 06:21:39 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46003 | Local Step: 9700 // 12101, total_eta: 81 days, 18:43:40, epoch_eta:4:02:53, time: all:6.151, data:0.001, vae:0.019, IQA:4.241, Connector:0.077, model:1.365, lr:1.414e-06, loss:0.009loss:0.2717, grad_norm:0.5809 | |
| 2025-10-27 06:31:44 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46103 | Local Step: 9800 // 12101, total_eta: 81 days, 18:28:49, epoch_eta:3:52:46, time: all:6.046, data:0.001, vae:0.018, IQA:4.317, Connector:0.068, model:1.271, lr:1.414e-06, loss:0.559loss:0.2793, grad_norm:0.5524 | |
| 2025-10-27 06:41:48 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46203 | Local Step: 9900 // 12101, total_eta: 81 days, 18:13:28, epoch_eta:3:42:38, time: all:6.043, data:0.001, vae:0.018, IQA:4.347, Connector:0.066, model:1.244, lr:1.414e-06, loss:0.332loss:0.2683, grad_norm:0.4229 | |
| 2025-10-27 06:51:58 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46303 | Local Step: 10000 // 12101, total_eta: 81 days, 18:09:50, epoch_eta:3:32:32, time: all:6.103, data:0.001, vae:0.018, IQA:4.320, Connector:0.072, model:1.308, lr:1.414e-06, loss:0.258loss:0.2803, grad_norm:0.7715 | |
| 2025-10-27 07:02:04 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46403 | Local Step: 10100 // 12101, total_eta: 81 days, 17:57:38, epoch_eta:3:22:25, time: all:6.059, data:0.001, vae:0.018, IQA:4.287, Connector:0.069, model:1.301, lr:1.414e-06, loss:0.605loss:0.2792, grad_norm:0.5684 | |
| 2025-10-27 07:12:10 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46503 | Local Step: 10200 // 12101, total_eta: 81 days, 17:46:08, epoch_eta:3:12:17, time: all:6.062, data:0.001, vae:0.018, IQA:4.268, Connector:0.072, model:1.317, lr:1.414e-06, loss:0.009loss:0.2650, grad_norm:0.5264 | |
| 2025-10-27 07:22:08 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46603 | Local Step: 10300 // 12101, total_eta: 81 days, 17:18:40, epoch_eta:3:02:09, time: all:5.977, data:0.001, vae:0.019, IQA:4.345, Connector:0.068, model:1.185, lr:1.414e-06, loss:0.156loss:0.2749, grad_norm:0.7415 | |
| 2025-10-27 07:32:14 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46703 | Local Step: 10400 // 12101, total_eta: 81 days, 17:06:10, epoch_eta:2:52:02, time: all:6.056, data:0.001, vae:0.018, IQA:4.415, Connector:0.068, model:1.186, lr:1.414e-06, loss:0.375loss:0.2902, grad_norm:0.7126 | |
| 2025-10-27 07:42:19 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46803 | Local Step: 10500 // 12101, total_eta: 81 days, 16:53:26, epoch_eta:2:41:55, time: all:6.054, data:0.001, vae:0.018, IQA:4.300, Connector:0.067, model:1.297, lr:1.414e-06, loss:0.305loss:0.2829, grad_norm:0.6166 | |
| 2025-10-27 07:52:17 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 46903 | Local Step: 10600 // 12101, total_eta: 81 days, 16:27:53, epoch_eta:2:31:47, time: all:5.984, data:0.001, vae:0.018, IQA:4.234, Connector:0.070, model:1.276, lr:1.414e-06, loss:0.178loss:0.2712, grad_norm:0.4831 | |
| 2025-10-27 08:02:39 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47003 | Local Step: 10700 // 12101, total_eta: 81 days, 16:45:17, epoch_eta:2:21:42, time: all:6.219, data:0.001, vae:0.019, IQA:4.342, Connector:0.076, model:1.322, lr:1.414e-06, loss:0.279loss:0.2693, grad_norm:1.2477 | |
| 2025-10-27 08:12:45 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47103 | Local Step: 10800 // 12101, total_eta: 81 days, 16:33:23, epoch_eta:2:11:35, time: all:6.059, data:0.001, vae:0.018, IQA:4.252, Connector:0.069, model:1.335, lr:1.414e-06, loss:0.320loss:0.2584, grad_norm:0.9678 | |
| 2025-10-27 08:22:50 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47203 | Local Step: 10900 // 12101, total_eta: 81 days, 16:18:56, epoch_eta:2:01:28, time: all:6.044, data:0.001, vae:0.018, IQA:4.305, Connector:0.068, model:1.275, lr:1.414e-06, loss:0.457loss:0.2872, grad_norm:0.4076 | |
| 2025-10-27 08:32:56 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47303 | Local Step: 11000 // 12101, total_eta: 81 days, 16:06:56, epoch_eta:1:51:21, time: all:6.058, data:0.001, vae:0.018, IQA:4.258, Connector:0.070, model:1.324, lr:1.414e-06, loss:0.723loss:0.2760, grad_norm:0.4009 | |
| 2025-10-27 08:42:59 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47403 | Local Step: 11100 // 12101, total_eta: 81 days, 15:50:13, epoch_eta:1:41:14, time: all:6.031, data:0.001, vae:0.018, IQA:4.349, Connector:0.069, model:1.224, lr:1.414e-06, loss:0.859loss:0.2759, grad_norm:0.4740 | |
| 2025-10-27 08:52:58 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47503 | Local Step: 11200 // 12101, total_eta: 81 days, 15:27:36, epoch_eta:1:31:06, time: all:5.996, data:0.001, vae:0.018, IQA:4.266, Connector:0.069, model:1.259, lr:1.414e-06, loss:0.123loss:0.2809, grad_norm:0.4769 | |
| 2025-10-27 09:03:01 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47603 | Local Step: 11300 // 12101, total_eta: 81 days, 15:11:13, epoch_eta:1:20:59, time: all:6.031, data:0.001, vae:0.018, IQA:4.242, Connector:0.068, model:1.319, lr:1.414e-06, loss:0.089loss:0.2702, grad_norm:0.3814 | |
| 2025-10-27 09:13:04 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47703 | Local Step: 11400 // 12101, total_eta: 81 days, 14:54:15, epoch_eta:1:10:52, time: all:6.027, data:0.001, vae:0.018, IQA:4.348, Connector:0.068, model:1.211, lr:1.414e-06, loss:0.145loss:0.2646, grad_norm:0.4355 | |
| 2025-10-27 09:23:07 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47803 | Local Step: 11500 // 12101, total_eta: 81 days, 14:37:33, epoch_eta:1:00:45, time: all:6.028, data:0.001, vae:0.018, IQA:4.253, Connector:0.072, model:1.303, lr:1.414e-06, loss:0.185loss:0.2500, grad_norm:0.3299 | |
| 2025-10-27 09:33:16 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 47903 | Local Step: 11600 // 12101, total_eta: 81 days, 14:31:13, epoch_eta:0:50:39, time: all:6.089, data:0.001, vae:0.018, IQA:4.325, Connector:0.071, model:1.292, lr:1.414e-06, loss:0.150loss:0.2752, grad_norm:0.5461 | |
| 2025-10-27 09:43:31 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 48003 | Local Step: 11700 // 12101, total_eta: 81 days, 14:35:11, epoch_eta:0:40:33, time: all:6.152, data:0.001, vae:0.019, IQA:4.252, Connector:0.079, model:1.354, lr:1.414e-06, loss:0.332loss:0.2805, grad_norm:0.4777 | |
| 2025-10-27 09:53:33 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 48103 | Local Step: 11800 // 12101, total_eta: 81 days, 14:17:11, epoch_eta:0:30:26, time: all:6.019, data:0.001, vae:0.018, IQA:4.241, Connector:0.070, model:1.305, lr:1.414e-06, loss:0.011loss:0.2712, grad_norm:0.3971 | |
| 2025-10-27 10:03:34 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 48203 | Local Step: 11900 // 12101, total_eta: 81 days, 13:57:45, epoch_eta:0:20:19, time: all:6.010, data:0.001, vae:0.018, IQA:4.294, Connector:0.067, model:1.250, lr:1.414e-06, loss:0.148loss:0.2649, grad_norm:0.4383 | |
| 2025-10-27 10:13:37 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 48303 | Local Step: 12000 // 12101, total_eta: 81 days, 13:41:41, epoch_eta:0:10:12, time: all:6.030, data:0.001, vae:0.018, IQA:4.305, Connector:0.069, model:1.262, lr:1.414e-06, loss:0.303loss:0.2887, grad_norm:0.3600 | |
| 2025-10-27 10:23:43 - [1m[Sana][0m - INFO - Epoch: 4 | Global Step: 48403 | Local Step: 12100 // 12101, total_eta: 81 days, 13:30:31, epoch_eta:0:00:06, time: all:6.060, data:0.001, vae:0.018, IQA:4.341, Connector:0.067, model:1.271, lr:1.414e-06, loss:0.213loss:0.2787, grad_norm:0.3664 | |
| 2025-10-27 10:23:55 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 4 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_4_step_48405_weight.pth. | |
| 2025-10-27 10:24:25 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 4 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_4_step_48405.pth. | |
| 2025-10-27 10:24:33 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 48405 | Local Step: 1 // 12101, total_eta: 107 days, 0:23:48, epoch_eta:1 day, 2:45:07, time: all:0.080, data:0.017, vae:0.000, IQA:0.038, Connector:0.001, model:0.017, lr:1.414e-06, loss:0.178loss:0.2207 | |
| 2025-10-27 10:34:44 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 48504 | Local Step: 100 // 12101, total_eta: 83 days, 5:51:20, epoch_eta:20:38:26, time: all:6.112, data:0.001, vae:0.019, IQA:4.206, Connector:0.075, model:1.365, lr:1.414e-06, loss:0.152loss:0.2881, grad_norm:0.4505 | |
| 2025-10-27 10:44:47 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 48604 | Local Step: 200 // 12101, total_eta: 82 days, 3:18:58, epoch_eta:20:11:55, time: all:6.028, data:0.001, vae:0.019, IQA:4.268, Connector:0.070, model:1.286, lr:1.414e-06, loss:0.324loss:0.2608, grad_norm:0.3590 | |
| 2025-10-27 10:54:48 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 48704 | Local Step: 300 // 12101, total_eta: 81 days, 16:45:00, epoch_eta:19:55:23, time: all:6.013, data:0.001, vae:0.018, IQA:4.291, Connector:0.070, model:1.267, lr:1.414e-06, loss:0.469loss:0.2720, grad_norm:0.5481 | |
| 2025-10-27 11:04:52 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 48804 | Local Step: 400 // 12101, total_eta: 81 days, 13:37:10, epoch_eta:19:43:28, time: all:6.041, data:0.001, vae:0.018, IQA:4.367, Connector:0.066, model:1.215, lr:1.414e-06, loss:0.060loss:0.2847, grad_norm:0.8190 | |
| 2025-10-27 11:15:00 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 48904 | Local Step: 500 // 12101, total_eta: 81 days, 13:50:33, epoch_eta:19:33:35, time: all:6.075, data:0.001, vae:0.018, IQA:4.374, Connector:0.065, model:1.241, lr:1.414e-06, loss:0.087loss:0.2784, grad_norm:0.4834 | |
| 2025-10-27 11:25:17 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49004 | Local Step: 600 // 12101, total_eta: 81 days, 19:09:54, epoch_eta:19:26:44, time: all:6.172, data:0.001, vae:0.019, IQA:4.271, Connector:0.076, model:1.346, lr:1.414e-06, loss:0.273loss:0.2536, grad_norm:0.4167 | |
| 2025-10-27 11:35:20 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49104 | Local Step: 700 // 12101, total_eta: 81 days, 16:27:50, epoch_eta:19:15:06, time: all:6.032, data:0.001, vae:0.018, IQA:4.237, Connector:0.068, model:1.329, lr:1.414e-06, loss:0.445loss:0.2761, grad_norm:0.3721 | |
| 2025-10-27 11:45:23 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49204 | Local Step: 800 // 12101, total_eta: 81 days, 14:29:42, epoch_eta:19:03:55, time: all:6.034, data:0.001, vae:0.019, IQA:4.284, Connector:0.069, model:1.276, lr:1.414e-06, loss:0.089loss:0.2800, grad_norm:0.3133 | |
| 2025-10-27 11:55:25 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49304 | Local Step: 900 // 12101, total_eta: 81 days, 12:14:16, epoch_eta:18:52:35, time: all:6.015, data:0.001, vae:0.018, IQA:4.250, Connector:0.070, model:1.309, lr:1.414e-06, loss:0.926loss:0.2665, grad_norm:0.4518 | |
| 2025-10-27 12:05:28 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49404 | Local Step: 1000 // 12101, total_eta: 81 days, 10:52:54, epoch_eta:18:41:48, time: all:6.030, data:0.001, vae:0.019, IQA:4.268, Connector:0.069, model:1.289, lr:1.414e-06, loss:0.523loss:0.2715, grad_norm:0.4307 | |
| 2025-10-27 12:15:25 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49504 | Local Step: 1100 // 12101, total_eta: 81 days, 8:02:36, epoch_eta:18:30:10, time: all:5.972, data:0.001, vae:0.019, IQA:4.235, Connector:0.071, model:1.269, lr:1.414e-06, loss:0.277loss:0.2649, grad_norm:0.7320 | |
| 2025-10-27 12:25:28 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49604 | Local Step: 1200 // 12101, total_eta: 81 days, 7:08:07, epoch_eta:18:19:40, time: all:6.027, data:0.001, vae:0.019, IQA:4.288, Connector:0.070, model:1.269, lr:1.414e-06, loss:0.586loss:0.2694, grad_norm:0.4266 | |
| 2025-10-27 12:35:30 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49704 | Local Step: 1300 // 12101, total_eta: 81 days, 6:10:42, epoch_eta:18:09:08, time: all:6.021, data:0.001, vae:0.018, IQA:4.337, Connector:0.068, model:1.236, lr:1.414e-06, loss:0.279loss:0.2862, grad_norm:0.3654 | |
| 2025-10-27 12:45:32 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49804 | Local Step: 1400 // 12101, total_eta: 81 days, 5:19:12, epoch_eta:17:58:40, time: all:6.020, data:0.001, vae:0.018, IQA:4.297, Connector:0.068, model:1.257, lr:1.414e-06, loss:0.089loss:0.2824, grad_norm:0.6963 | |
| 2025-10-27 12:55:28 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 49904 | Local Step: 1500 // 12101, total_eta: 81 days, 3:15:39, epoch_eta:17:47:33, time: all:5.960, data:0.001, vae:0.018, IQA:4.147, Connector:0.071, model:1.334, lr:1.414e-06, loss:0.236loss:0.2804, grad_norm:0.5829 | |
| 2025-10-27 13:05:04 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_50000_weight.pth. | |
| 2025-10-27 13:05:36 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_50000.pth. | |
| 2025-10-27 13:06:23 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50004 | Local Step: 1600 // 12101, total_eta: 81 days, 13:13:23, epoch_eta:17:42:59, time: all:6.545, data:0.001, vae:0.020, IQA:4.284, Connector:0.074, model:1.357, lr:1.414e-06, loss:0.067loss:0.2721, grad_norm:0.4421 | |
| 2025-10-27 13:16:28 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50104 | Local Step: 1700 // 12101, total_eta: 81 days, 12:37:34, epoch_eta:17:32:38, time: all:6.051, data:0.001, vae:0.018, IQA:4.241, Connector:0.071, model:1.327, lr:1.414e-06, loss:0.750loss:0.2859, grad_norm:1.7383 | |
| 2025-10-27 13:26:30 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50204 | Local Step: 1800 // 12101, total_eta: 81 days, 11:30:53, epoch_eta:17:22:00, time: all:6.020, data:0.001, vae:0.018, IQA:4.342, Connector:0.067, model:1.219, lr:1.414e-06, loss:0.016loss:0.2575, grad_norm:0.5107 | |
| 2025-10-27 13:36:37 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50304 | Local Step: 1900 // 12101, total_eta: 81 days, 11:26:24, epoch_eta:17:11:56, time: all:6.075, data:0.001, vae:0.018, IQA:4.289, Connector:0.070, model:1.313, lr:1.414e-06, loss:0.109loss:0.2800, grad_norm:0.4595 | |
| 2025-10-27 13:46:41 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50404 | Local Step: 2000 // 12101, total_eta: 81 days, 10:42:37, epoch_eta:17:01:32, time: all:6.035, data:0.001, vae:0.018, IQA:4.310, Connector:0.069, model:1.258, lr:1.414e-06, loss:0.879loss:0.2846, grad_norm:0.5770 | |
| 2025-10-27 13:56:44 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50504 | Local Step: 2100 // 12101, total_eta: 81 days, 9:59:36, epoch_eta:16:51:08, time: all:6.032, data:0.001, vae:0.018, IQA:4.206, Connector:0.070, model:1.351, lr:1.414e-06, loss:0.028loss:0.2721, grad_norm:0.7399 | |
| 2025-10-27 14:06:44 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50604 | Local Step: 2200 // 12101, total_eta: 81 days, 8:56:29, epoch_eta:16:40:34, time: all:6.006, data:0.001, vae:0.018, IQA:4.278, Connector:0.067, model:1.266, lr:1.414e-06, loss:0.113loss:0.2953, grad_norm:0.4253 | |
| 2025-10-27 14:16:47 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50704 | Local Step: 2300 // 12101, total_eta: 81 days, 8:16:49, epoch_eta:16:30:13, time: all:6.028, data:0.001, vae:0.018, IQA:4.351, Connector:0.069, model:1.208, lr:1.414e-06, loss:3.984loss:0.2808, grad_norm:0.4406 | |
| 2025-10-27 14:26:48 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50804 | Local Step: 2400 // 12101, total_eta: 81 days, 7:26:15, epoch_eta:16:19:46, time: all:6.012, data:0.001, vae:0.018, IQA:4.277, Connector:0.069, model:1.262, lr:1.414e-06, loss:0.367loss:0.2652, grad_norm:0.6094 | |
| 2025-10-27 14:36:49 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 50904 | Local Step: 2500 // 12101, total_eta: 81 days, 6:37:18, epoch_eta:16:09:21, time: all:6.010, data:0.001, vae:0.019, IQA:4.293, Connector:0.068, model:1.246, lr:1.414e-06, loss:0.297loss:0.2700, grad_norm:0.4302 | |
| 2025-10-27 14:47:09 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51004 | Local Step: 2600 // 12101, total_eta: 81 days, 8:10:23, epoch_eta:16:00:06, time: all:6.197, data:0.001, vae:0.020, IQA:4.276, Connector:0.078, model:1.381, lr:1.414e-06, loss:0.101loss:0.2615, grad_norm:0.3182 | |
| 2025-10-27 14:57:09 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51104 | Local Step: 2700 // 12101, total_eta: 81 days, 7:14:17, epoch_eta:15:49:37, time: all:5.999, data:0.001, vae:0.018, IQA:4.260, Connector:0.070, model:1.268, lr:1.414e-06, loss:0.132loss:0.2667, grad_norm:0.6122 | |
| 2025-10-27 15:07:12 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51204 | Local Step: 2800 // 12101, total_eta: 81 days, 6:43:03, epoch_eta:15:39:21, time: all:6.030, data:0.001, vae:0.018, IQA:4.286, Connector:0.069, model:1.275, lr:1.414e-06, loss:0.161loss:0.2630, grad_norm:0.4476 | |
| 2025-10-27 15:17:16 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51304 | Local Step: 2900 // 12101, total_eta: 81 days, 6:23:32, epoch_eta:15:29:11, time: all:6.046, data:0.001, vae:0.018, IQA:4.333, Connector:0.068, model:1.255, lr:1.414e-06, loss:0.201loss:0.2586, grad_norm:0.3893 | |
| 2025-10-27 15:27:27 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51404 | Local Step: 3000 // 12101, total_eta: 81 days, 6:41:04, epoch_eta:15:19:18, time: all:6.102, data:0.001, vae:0.018, IQA:4.350, Connector:0.067, model:1.287, lr:1.414e-06, loss:0.196loss:0.2687, grad_norm:0.5505 | |
| 2025-10-27 15:37:27 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51504 | Local Step: 3100 // 12101, total_eta: 81 days, 5:54:13, epoch_eta:15:08:54, time: all:6.002, data:0.001, vae:0.019, IQA:4.270, Connector:0.068, model:1.264, lr:1.414e-06, loss:0.379loss:0.2718, grad_norm:0.7910 | |
| 2025-10-27 15:47:28 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51604 | Local Step: 3200 // 12101, total_eta: 81 days, 5:13:34, epoch_eta:14:58:34, time: all:6.008, data:0.001, vae:0.018, IQA:4.341, Connector:0.065, model:1.215, lr:1.414e-06, loss:0.354loss:0.2842, grad_norm:0.4528 | |
| 2025-10-27 15:57:29 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51704 | Local Step: 3300 // 12101, total_eta: 81 days, 4:35:49, epoch_eta:14:48:16, time: all:6.010, data:0.001, vae:0.018, IQA:4.275, Connector:0.068, model:1.261, lr:1.414e-06, loss:0.142loss:0.2732, grad_norm:0.4530 | |
| 2025-10-27 16:07:37 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51804 | Local Step: 3400 // 12101, total_eta: 81 days, 4:42:00, epoch_eta:14:38:18, time: all:6.084, data:0.001, vae:0.019, IQA:4.270, Connector:0.071, model:1.326, lr:1.414e-06, loss:0.193loss:0.2724, grad_norm:0.5086 | |
| 2025-10-27 16:17:44 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 51904 | Local Step: 3500 // 12101, total_eta: 81 days, 4:39:49, epoch_eta:14:28:16, time: all:6.071, data:0.001, vae:0.019, IQA:4.360, Connector:0.069, model:1.255, lr:1.414e-06, loss:0.160loss:0.2825, grad_norm:0.4477 | |
| 2025-10-27 16:28:09 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52004 | Local Step: 3600 // 12101, total_eta: 81 days, 6:13:04, epoch_eta:14:18:56, time: all:6.250, data:0.001, vae:0.020, IQA:4.333, Connector:0.079, model:1.371, lr:1.414e-06, loss:0.586loss:0.2701, grad_norm:0.4386 | |
| 2025-10-27 16:38:12 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52104 | Local Step: 3700 // 12101, total_eta: 81 days, 5:44:14, epoch_eta:14:08:41, time: all:6.026, data:0.001, vae:0.019, IQA:4.396, Connector:0.067, model:1.176, lr:1.414e-06, loss:0.244loss:0.2749, grad_norm:0.3647 | |
| 2025-10-27 16:48:10 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52204 | Local Step: 3800 // 12101, total_eta: 81 days, 4:51:19, epoch_eta:13:58:17, time: all:5.977, data:0.001, vae:0.018, IQA:4.226, Connector:0.069, model:1.282, lr:1.414e-06, loss:0.379loss:0.2704, grad_norm:0.3636 | |
| 2025-10-27 16:58:11 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52304 | Local Step: 3900 // 12101, total_eta: 81 days, 4:17:54, epoch_eta:13:48:01, time: all:6.012, data:0.001, vae:0.019, IQA:4.289, Connector:0.069, model:1.261, lr:1.414e-06, loss:0.146loss:0.2785, grad_norm:0.3927 | |
| 2025-10-27 17:08:12 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52404 | Local Step: 4000 // 12101, total_eta: 81 days, 3:47:35, epoch_eta:13:37:47, time: all:6.016, data:0.001, vae:0.018, IQA:4.366, Connector:0.069, model:1.193, lr:1.414e-06, loss:0.014loss:0.2695, grad_norm:0.3242 | |
| 2025-10-27 17:18:20 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52504 | Local Step: 4100 // 12101, total_eta: 81 days, 3:45:38, epoch_eta:13:27:44, time: all:6.074, data:0.001, vae:0.018, IQA:4.260, Connector:0.068, model:1.327, lr:1.414e-06, loss:0.127loss:0.2743, grad_norm:0.6221 | |
| 2025-10-27 17:28:24 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52604 | Local Step: 4200 // 12101, total_eta: 81 days, 3:27:15, epoch_eta:13:17:35, time: all:6.039, data:0.001, vae:0.019, IQA:4.278, Connector:0.067, model:1.302, lr:1.414e-06, loss:0.084loss:0.2863, grad_norm:0.6585 | |
| 2025-10-27 17:38:30 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52704 | Local Step: 4300 // 12101, total_eta: 81 days, 3:18:43, epoch_eta:13:07:30, time: all:6.060, data:0.001, vae:0.018, IQA:4.329, Connector:0.069, model:1.262, lr:1.414e-06, loss:0.141loss:0.2680, grad_norm:0.3560 | |
| 2025-10-27 17:48:31 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52804 | Local Step: 4400 // 12101, total_eta: 81 days, 2:50:23, epoch_eta:12:57:17, time: all:6.015, data:0.001, vae:0.019, IQA:4.278, Connector:0.067, model:1.272, lr:1.414e-06, loss:0.232loss:0.2609, grad_norm:0.3823 | |
| 2025-10-27 17:58:34 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 52904 | Local Step: 4500 // 12101, total_eta: 81 days, 2:28:01, epoch_eta:12:47:07, time: all:6.027, data:0.001, vae:0.018, IQA:4.247, Connector:0.070, model:1.317, lr:1.414e-06, loss:0.361loss:0.2671, grad_norm:0.5446 | |
| 2025-10-27 18:08:54 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53004 | Local Step: 4600 // 12101, total_eta: 81 days, 3:20:31, epoch_eta:12:37:25, time: all:6.205, data:0.001, vae:0.019, IQA:4.378, Connector:0.074, model:1.283, lr:1.414e-06, loss:0.199loss:0.2599, grad_norm:0.4585 | |
| 2025-10-27 18:18:53 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53104 | Local Step: 4700 // 12101, total_eta: 81 days, 2:42:07, epoch_eta:12:27:09, time: all:5.990, data:0.001, vae:0.018, IQA:4.231, Connector:0.069, model:1.292, lr:1.414e-06, loss:0.135loss:0.2751, grad_norm:0.4110 | |
| 2025-10-27 18:29:00 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53204 | Local Step: 4800 // 12101, total_eta: 81 days, 2:33:26, epoch_eta:12:17:04, time: all:6.061, data:0.001, vae:0.018, IQA:4.352, Connector:0.069, model:1.244, lr:1.414e-06, loss:0.144loss:0.2538, grad_norm:0.4889 | |
| 2025-10-27 18:39:04 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53304 | Local Step: 4900 // 12101, total_eta: 81 days, 2:17:35, epoch_eta:12:06:56, time: all:6.043, data:0.001, vae:0.018, IQA:4.355, Connector:0.068, model:1.228, lr:1.414e-06, loss:0.141loss:0.2660, grad_norm:0.3924 | |
| 2025-10-27 18:49:01 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53404 | Local Step: 5000 // 12101, total_eta: 81 days, 1:34:30, epoch_eta:11:56:38, time: all:5.971, data:0.001, vae:0.018, IQA:4.201, Connector:0.068, model:1.298, lr:1.414e-06, loss:0.021loss:0.2734, grad_norm:0.2959 | |
| 2025-10-27 18:59:04 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53504 | Local Step: 5100 // 12101, total_eta: 81 days, 1:13:46, epoch_eta:11:46:28, time: all:6.027, data:0.001, vae:0.018, IQA:4.264, Connector:0.067, model:1.293, lr:1.414e-06, loss:0.672loss:0.2667, grad_norm:0.9732 | |
| 2025-10-27 19:09:10 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53604 | Local Step: 5200 // 12101, total_eta: 81 days, 1:05:33, epoch_eta:11:36:24, time: all:6.060, data:0.001, vae:0.019, IQA:4.306, Connector:0.069, model:1.288, lr:1.414e-06, loss:0.008loss:0.2533, grad_norm:0.5889 | |
| 2025-10-27 19:19:15 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53704 | Local Step: 5300 // 12101, total_eta: 81 days, 0:55:18, epoch_eta:11:26:18, time: all:6.054, data:0.001, vae:0.018, IQA:4.184, Connector:0.070, model:1.387, lr:1.414e-06, loss:0.352loss:0.2643, grad_norm:0.4255 | |
| 2025-10-27 19:29:19 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53804 | Local Step: 5400 // 12101, total_eta: 81 days, 0:40:08, epoch_eta:11:16:11, time: all:6.041, data:0.001, vae:0.018, IQA:4.332, Connector:0.069, model:1.236, lr:1.414e-06, loss:0.719loss:0.2843, grad_norm:0.5016 | |
| 2025-10-27 19:39:32 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 53904 | Local Step: 5500 // 12101, total_eta: 81 days, 0:55:09, epoch_eta:11:06:14, time: all:6.126, data:0.001, vae:0.018, IQA:4.357, Connector:0.068, model:1.299, lr:1.414e-06, loss:0.293loss:0.2751, grad_norm:0.3956 | |
| 2025-10-27 19:49:53 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54004 | Local Step: 5600 // 12101, total_eta: 81 days, 1:39:22, epoch_eta:10:56:27, time: all:6.214, data:0.001, vae:0.020, IQA:4.333, Connector:0.077, model:1.324, lr:1.414e-06, loss:0.058loss:0.2769, grad_norm:0.9147 | |
| 2025-10-27 19:59:54 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54104 | Local Step: 5700 // 12101, total_eta: 81 days, 1:13:00, epoch_eta:10:46:15, time: all:6.010, data:0.001, vae:0.018, IQA:4.229, Connector:0.068, model:1.320, lr:1.414e-06, loss:0.789loss:0.2637, grad_norm:0.4017 | |
| 2025-10-27 20:09:57 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54204 | Local Step: 5800 // 12101, total_eta: 81 days, 0:54:08, epoch_eta:10:36:07, time: all:6.031, data:0.001, vae:0.018, IQA:4.316, Connector:0.069, model:1.250, lr:1.414e-06, loss:0.213loss:0.2840, grad_norm:0.4212 | |
| 2025-10-27 20:20:07 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54304 | Local Step: 5900 // 12101, total_eta: 81 days, 0:56:25, epoch_eta:10:26:05, time: all:6.095, data:0.001, vae:0.018, IQA:4.401, Connector:0.068, model:1.232, lr:1.414e-06, loss:0.352loss:0.2698, grad_norm:0.3856 | |
| 2025-10-27 20:30:12 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54404 | Local Step: 6000 // 12101, total_eta: 81 days, 0:45:55, epoch_eta:10:15:59, time: all:6.057, data:0.001, vae:0.018, IQA:4.327, Connector:0.069, model:1.267, lr:1.414e-06, loss:0.096loss:0.2760, grad_norm:0.4717 | |
| 2025-10-27 20:40:15 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54504 | Local Step: 6100 // 12101, total_eta: 81 days, 0:25:20, epoch_eta:10:05:50, time: all:6.025, data:0.001, vae:0.018, IQA:4.281, Connector:0.071, model:1.265, lr:1.414e-06, loss:0.295loss:0.2809, grad_norm:0.4315 | |
| 2025-10-27 20:50:24 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54604 | Local Step: 6200 // 12101, total_eta: 81 days, 0:25:38, epoch_eta:9:55:48, time: all:6.091, data:0.001, vae:0.018, IQA:4.310, Connector:0.072, model:1.289, lr:1.414e-06, loss:0.447loss:0.2620, grad_norm:0.6296 | |
| 2025-10-27 21:00:27 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54704 | Local Step: 6300 // 12101, total_eta: 81 days, 0:06:19, epoch_eta:9:45:39, time: all:6.028, data:0.001, vae:0.018, IQA:4.253, Connector:0.071, model:1.303, lr:1.414e-06, loss:0.859loss:0.2710, grad_norm:0.4270 | |
| 2025-10-27 21:10:39 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54804 | Local Step: 6400 // 12101, total_eta: 81 days, 0:14:59, epoch_eta:9:35:39, time: all:6.120, data:0.001, vae:0.018, IQA:4.432, Connector:0.066, model:1.218, lr:1.414e-06, loss:0.250loss:0.2756, grad_norm:0.4927 | |
| 2025-10-27 21:20:42 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 54904 | Local Step: 6500 // 12101, total_eta: 80 days, 23:57:51, epoch_eta:9:25:31, time: all:6.035, data:0.001, vae:0.018, IQA:4.236, Connector:0.070, model:1.318, lr:1.414e-06, loss:0.176loss:0.2790, grad_norm:0.5778 | |
| 2025-10-27 21:30:21 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_55000_weight.pth. | |
| 2025-10-27 21:30:51 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_55000.pth. | |
| 2025-10-27 21:31:39 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55004 | Local Step: 6600 // 12101, total_eta: 81 days, 2:16:06, epoch_eta:9:16:07, time: all:6.567, data:0.001, vae:0.019, IQA:4.315, Connector:0.078, model:1.337, lr:1.414e-06, loss:0.285loss:0.2513, grad_norm:0.4141 | |
| 2025-10-27 21:41:41 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55104 | Local Step: 6700 // 12101, total_eta: 81 days, 1:53:16, epoch_eta:9:05:57, time: all:6.021, data:0.001, vae:0.018, IQA:4.300, Connector:0.069, model:1.233, lr:1.414e-06, loss:0.149loss:0.2632, grad_norm:0.4671 | |
| 2025-10-27 21:51:47 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55204 | Local Step: 6800 // 12101, total_eta: 81 days, 1:41:24, epoch_eta:8:55:50, time: all:6.059, data:0.001, vae:0.018, IQA:4.352, Connector:0.066, model:1.225, lr:1.414e-06, loss:0.189loss:0.2855, grad_norm:0.4674 | |
| 2025-10-27 22:01:53 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55304 | Local Step: 6900 // 12101, total_eta: 81 days, 1:29:12, epoch_eta:8:45:43, time: all:6.058, data:0.001, vae:0.018, IQA:4.328, Connector:0.068, model:1.254, lr:1.414e-06, loss:0.008loss:0.2589, grad_norm:0.3719 | |
| 2025-10-27 22:12:02 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55404 | Local Step: 7000 // 12101, total_eta: 81 days, 1:25:33, epoch_eta:8:35:38, time: all:6.088, data:0.001, vae:0.018, IQA:4.295, Connector:0.068, model:1.304, lr:1.414e-06, loss:0.160loss:0.2573, grad_norm:0.3901 | |
| 2025-10-27 22:22:08 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55504 | Local Step: 7100 // 12101, total_eta: 81 days, 1:15:11, epoch_eta:8:25:32, time: all:6.064, data:0.001, vae:0.018, IQA:4.262, Connector:0.071, model:1.308, lr:1.414e-06, loss:0.123loss:0.2741, grad_norm:0.3820 | |
| 2025-10-27 22:32:16 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55604 | Local Step: 7200 // 12101, total_eta: 81 days, 1:09:07, epoch_eta:8:15:26, time: all:6.080, data:0.001, vae:0.018, IQA:4.290, Connector:0.070, model:1.306, lr:1.414e-06, loss:0.170loss:0.2667, grad_norm:0.5312 | |
| 2025-10-27 22:42:19 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55704 | Local Step: 7300 // 12101, total_eta: 81 days, 0:48:21, epoch_eta:8:05:17, time: all:6.025, data:0.001, vae:0.018, IQA:4.280, Connector:0.068, model:1.275, lr:1.414e-06, loss:0.279loss:0.2653, grad_norm:0.5067 | |
| 2025-10-27 22:52:27 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55804 | Local Step: 7400 // 12101, total_eta: 81 days, 0:43:36, epoch_eta:7:55:12, time: all:6.085, data:0.001, vae:0.019, IQA:4.313, Connector:0.068, model:1.307, lr:1.414e-06, loss:0.305loss:0.2811, grad_norm:0.3506 | |
| 2025-10-27 23:02:32 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 55904 | Local Step: 7500 // 12101, total_eta: 81 days, 0:29:43, epoch_eta:7:45:05, time: all:6.050, data:0.001, vae:0.018, IQA:4.316, Connector:0.068, model:1.269, lr:1.414e-06, loss:0.050loss:0.2835, grad_norm:0.6345 | |
| 2025-10-27 23:12:54 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56004 | Local Step: 7600 // 12101, total_eta: 81 days, 0:59:20, epoch_eta:7:35:07, time: all:6.222, data:0.001, vae:0.019, IQA:4.360, Connector:0.077, model:1.319, lr:1.414e-06, loss:0.234loss:0.2716, grad_norm:0.5804 | |
| 2025-10-27 23:22:53 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56104 | Local Step: 7700 // 12101, total_eta: 81 days, 0:28:34, epoch_eta:7:24:56, time: all:5.984, data:0.001, vae:0.018, IQA:4.270, Connector:0.067, model:1.246, lr:1.414e-06, loss:0.867loss:0.2694, grad_norm:0.8698 | |
| 2025-10-27 23:32:58 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56204 | Local Step: 7800 // 12101, total_eta: 81 days, 0:15:00, epoch_eta:7:14:49, time: all:6.052, data:0.001, vae:0.018, IQA:4.404, Connector:0.065, model:1.192, lr:1.414e-06, loss:0.318loss:0.2731, grad_norm:0.5244 | |
| 2025-10-27 23:43:01 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56304 | Local Step: 7900 // 12101, total_eta: 80 days, 23:56:55, epoch_eta:7:04:40, time: all:6.033, data:0.001, vae:0.018, IQA:4.330, Connector:0.067, model:1.245, lr:1.414e-06, loss:0.672loss:0.2810, grad_norm:0.8538 | |
| 2025-10-27 23:53:05 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56404 | Local Step: 8000 // 12101, total_eta: 80 days, 23:40:35, epoch_eta:6:54:32, time: all:6.039, data:0.001, vae:0.018, IQA:4.328, Connector:0.071, model:1.248, lr:1.414e-06, loss:0.229loss:0.2743, grad_norm:0.3820 | |
| 2025-10-28 00:03:09 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56504 | Local Step: 8100 // 12101, total_eta: 80 days, 23:24:27, epoch_eta:6:44:25, time: all:6.040, data:0.001, vae:0.018, IQA:4.300, Connector:0.072, model:1.243, lr:1.414e-06, loss:0.750loss:0.2789, grad_norm:0.7241 | |
| 2025-10-28 00:13:07 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56604 | Local Step: 8200 // 12101, total_eta: 80 days, 22:53:31, epoch_eta:6:34:14, time: all:5.976, data:0.001, vae:0.018, IQA:4.295, Connector:0.070, model:1.188, lr:1.414e-06, loss:0.492loss:0.2669, grad_norm:0.4474 | |
| 2025-10-28 00:23:08 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56704 | Local Step: 8300 // 12101, total_eta: 80 days, 22:32:20, epoch_eta:6:24:05, time: all:6.016, data:0.001, vae:0.018, IQA:4.291, Connector:0.069, model:1.241, lr:1.414e-06, loss:0.258loss:0.2658, grad_norm:0.3962 | |
| 2025-10-28 00:33:13 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56804 | Local Step: 8400 // 12101, total_eta: 80 days, 22:18:07, epoch_eta:6:13:58, time: all:6.045, data:0.001, vae:0.017, IQA:4.224, Connector:0.070, model:1.342, lr:1.414e-06, loss:0.120loss:0.2749, grad_norm:0.3140 | |
| 2025-10-28 00:43:20 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 56904 | Local Step: 8500 // 12101, total_eta: 80 days, 22:10:22, epoch_eta:6:03:52, time: all:6.073, data:0.001, vae:0.018, IQA:4.287, Connector:0.068, model:1.309, lr:1.414e-06, loss:0.742loss:0.2726, grad_norm:0.5570 | |
| 2025-10-28 00:53:40 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57004 | Local Step: 8600 // 12101, total_eta: 80 days, 22:30:18, epoch_eta:5:53:52, time: all:6.197, data:0.001, vae:0.019, IQA:4.309, Connector:0.073, model:1.348, lr:1.414e-06, loss:0.221loss:0.2577, grad_norm:0.4596 | |
| 2025-10-28 01:03:45 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57104 | Local Step: 8700 // 12101, total_eta: 80 days, 22:16:10, epoch_eta:5:43:44, time: all:6.046, data:0.001, vae:0.018, IQA:4.279, Connector:0.068, model:1.312, lr:1.414e-06, loss:0.125loss:0.2882, grad_norm:0.4590 | |
| 2025-10-28 01:13:50 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57204 | Local Step: 8800 // 12101, total_eta: 80 days, 22:04:16, epoch_eta:5:33:38, time: all:6.056, data:0.001, vae:0.019, IQA:4.281, Connector:0.069, model:1.306, lr:1.414e-06, loss:0.344loss:0.2720, grad_norm:0.4098 | |
| 2025-10-28 01:23:53 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57304 | Local Step: 8900 // 12101, total_eta: 80 days, 21:46:29, epoch_eta:5:23:30, time: all:6.029, data:0.001, vae:0.018, IQA:4.321, Connector:0.068, model:1.246, lr:1.414e-06, loss:0.773loss:0.2671, grad_norm:0.3065 | |
| 2025-10-28 01:33:56 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57404 | Local Step: 9000 // 12101, total_eta: 80 days, 21:29:08, epoch_eta:5:13:22, time: all:6.030, data:0.001, vae:0.018, IQA:4.334, Connector:0.069, model:1.225, lr:1.414e-06, loss:0.139loss:0.2768, grad_norm:0.4886 | |
| 2025-10-28 01:44:03 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57504 | Local Step: 9100 // 12101, total_eta: 80 days, 21:20:05, epoch_eta:5:03:16, time: all:6.069, data:0.001, vae:0.018, IQA:4.296, Connector:0.067, model:1.326, lr:1.414e-06, loss:0.305loss:0.2961, grad_norm:0.3969 | |
| 2025-10-28 01:54:05 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57604 | Local Step: 9200 // 12101, total_eta: 80 days, 21:02:06, epoch_eta:4:53:09, time: all:6.026, data:0.001, vae:0.018, IQA:4.304, Connector:0.068, model:1.255, lr:1.414e-06, loss:0.166loss:0.2783, grad_norm:0.7734 | |
| 2025-10-28 02:04:04 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57704 | Local Step: 9300 // 12101, total_eta: 80 days, 20:35:20, epoch_eta:4:43:00, time: all:5.982, data:0.001, vae:0.019, IQA:4.266, Connector:0.066, model:1.268, lr:1.414e-06, loss:0.118loss:0.2550, grad_norm:0.3770 | |
| 2025-10-28 02:14:08 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57804 | Local Step: 9400 // 12101, total_eta: 80 days, 20:21:36, epoch_eta:4:32:53, time: all:6.044, data:0.001, vae:0.018, IQA:4.286, Connector:0.069, model:1.292, lr:1.414e-06, loss:0.828loss:0.2770, grad_norm:0.6632 | |
| 2025-10-28 02:24:10 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 57904 | Local Step: 9500 // 12101, total_eta: 80 days, 20:02:38, epoch_eta:4:22:46, time: all:6.018, data:0.001, vae:0.018, IQA:4.258, Connector:0.068, model:1.292, lr:1.414e-06, loss:0.104loss:0.2690, grad_norm:0.3758 | |
| 2025-10-28 02:34:31 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58004 | Local Step: 9600 // 12101, total_eta: 80 days, 20:21:38, epoch_eta:4:12:43, time: all:6.207, data:0.001, vae:0.019, IQA:4.383, Connector:0.073, model:1.286, lr:1.414e-06, loss:0.430loss:0.2731, grad_norm:0.4300 | |
| 2025-10-28 02:44:32 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58104 | Local Step: 9700 // 12101, total_eta: 80 days, 20:01:59, epoch_eta:4:02:36, time: all:6.015, data:0.001, vae:0.018, IQA:4.174, Connector:0.068, model:1.365, lr:1.414e-06, loss:0.195loss:0.2747, grad_norm:0.6276 | |
| 2025-10-28 02:54:34 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58204 | Local Step: 9800 // 12101, total_eta: 80 days, 19:43:58, epoch_eta:3:52:29, time: all:6.022, data:0.001, vae:0.019, IQA:4.286, Connector:0.068, model:1.276, lr:1.414e-06, loss:0.160loss:0.2779, grad_norm:0.4033 | |
| 2025-10-28 03:04:40 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58304 | Local Step: 9900 // 12101, total_eta: 80 days, 19:32:32, epoch_eta:3:42:22, time: all:6.055, data:0.001, vae:0.018, IQA:4.284, Connector:0.070, model:1.297, lr:1.414e-06, loss:0.256loss:0.2746, grad_norm:0.3403 | |
| 2025-10-28 03:14:45 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58404 | Local Step: 10000 // 12101, total_eta: 80 days, 19:19:32, epoch_eta:3:32:16, time: all:6.047, data:0.001, vae:0.019, IQA:4.245, Connector:0.070, model:1.335, lr:1.414e-06, loss:0.189loss:0.2724, grad_norm:0.4474 | |
| 2025-10-28 03:24:44 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58504 | Local Step: 10100 // 12101, total_eta: 80 days, 18:57:03, epoch_eta:3:22:08, time: all:5.997, data:0.001, vae:0.019, IQA:4.198, Connector:0.069, model:1.323, lr:1.414e-06, loss:0.244loss:0.2735, grad_norm:0.6914 | |
| 2025-10-28 03:34:50 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58604 | Local Step: 10200 // 12101, total_eta: 80 days, 18:46:54, epoch_eta:3:12:02, time: all:6.061, data:0.001, vae:0.019, IQA:4.365, Connector:0.068, model:1.239, lr:1.414e-06, loss:0.197loss:0.2701, grad_norm:1.0223 | |
| 2025-10-28 03:44:55 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58704 | Local Step: 10300 // 12101, total_eta: 80 days, 18:33:25, epoch_eta:3:01:56, time: all:6.043, data:0.001, vae:0.018, IQA:4.288, Connector:0.069, model:1.280, lr:1.414e-06, loss:0.103loss:0.2762, grad_norm:0.4802 | |
| 2025-10-28 03:54:55 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58804 | Local Step: 10400 // 12101, total_eta: 80 days, 18:11:49, epoch_eta:2:51:49, time: all:5.999, data:0.001, vae:0.018, IQA:4.265, Connector:0.068, model:1.266, lr:1.414e-06, loss:0.154loss:0.2571, grad_norm:0.5267 | |
| 2025-10-28 04:04:55 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 58904 | Local Step: 10500 // 12101, total_eta: 80 days, 17:51:36, epoch_eta:2:41:42, time: all:6.005, data:0.001, vae:0.019, IQA:4.262, Connector:0.067, model:1.285, lr:1.414e-06, loss:0.141loss:0.2756, grad_norm:0.4219 | |
| 2025-10-28 04:15:18 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59004 | Local Step: 10600 // 12101, total_eta: 80 days, 18:11:50, epoch_eta:2:31:38, time: all:6.228, data:0.001, vae:0.019, IQA:4.296, Connector:0.076, model:1.388, lr:1.414e-06, loss:0.008loss:0.2644, grad_norm:0.4445 | |
| 2025-10-28 04:25:20 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59104 | Local Step: 10700 // 12101, total_eta: 80 days, 17:53:34, epoch_eta:2:21:31, time: all:6.016, data:0.001, vae:0.019, IQA:4.240, Connector:0.069, model:1.302, lr:1.414e-06, loss:0.189loss:0.2723, grad_norm:0.5625 | |
| 2025-10-28 04:35:29 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59204 | Local Step: 10800 // 12101, total_eta: 80 days, 17:48:35, epoch_eta:2:11:25, time: all:6.090, data:0.001, vae:0.018, IQA:4.318, Connector:0.069, model:1.293, lr:1.414e-06, loss:0.166loss:0.2585, grad_norm:0.5579 | |
| 2025-10-28 04:45:28 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59304 | Local Step: 10900 // 12101, total_eta: 80 days, 17:25:53, epoch_eta:2:01:19, time: all:5.990, data:0.001, vae:0.018, IQA:4.311, Connector:0.067, model:1.220, lr:1.414e-06, loss:0.328loss:0.2771, grad_norm:0.5718 | |
| 2025-10-28 04:55:35 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59404 | Local Step: 11000 // 12101, total_eta: 80 days, 17:17:27, epoch_eta:1:51:13, time: all:6.070, data:0.001, vae:0.018, IQA:4.335, Connector:0.069, model:1.271, lr:1.414e-06, loss:0.179loss:0.2700, grad_norm:0.7425 | |
| 2025-10-28 05:05:38 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59504 | Local Step: 11100 // 12101, total_eta: 80 days, 17:02:57, epoch_eta:1:41:06, time: all:6.035, data:0.001, vae:0.019, IQA:4.255, Connector:0.071, model:1.305, lr:1.414e-06, loss:0.289loss:0.2757, grad_norm:0.3599 | |
| 2025-10-28 05:15:43 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59604 | Local Step: 11200 // 12101, total_eta: 80 days, 16:51:11, epoch_eta:1:31:00, time: all:6.051, data:0.001, vae:0.018, IQA:4.390, Connector:0.068, model:1.208, lr:1.414e-06, loss:0.143loss:0.2698, grad_norm:0.3158 | |
| 2025-10-28 05:25:47 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59704 | Local Step: 11300 // 12101, total_eta: 80 days, 16:36:53, epoch_eta:1:20:54, time: all:6.036, data:0.001, vae:0.018, IQA:4.269, Connector:0.068, model:1.311, lr:1.414e-06, loss:0.100loss:0.2641, grad_norm:0.3760 | |
| 2025-10-28 05:35:50 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59804 | Local Step: 11400 // 12101, total_eta: 80 days, 16:21:43, epoch_eta:1:10:48, time: all:6.030, data:0.001, vae:0.019, IQA:4.319, Connector:0.066, model:1.258, lr:1.414e-06, loss:0.414loss:0.2639, grad_norm:0.5322 | |
| 2025-10-28 05:45:56 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 59904 | Local Step: 11500 // 12101, total_eta: 80 days, 16:12:41, epoch_eta:1:00:42, time: all:6.066, data:0.001, vae:0.018, IQA:4.355, Connector:0.068, model:1.232, lr:1.414e-06, loss:0.211loss:0.2874, grad_norm:0.3793 | |
| 2025-10-28 05:55:46 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_60000_weight.pth. | |
| 2025-10-28 05:56:15 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_60000.pth. | |
| 2025-10-28 05:57:03 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 60004 | Local Step: 11600 // 12101, total_eta: 80 days, 17:42:09, epoch_eta:0:50:38, time: all:6.663, data:0.001, vae:0.019, IQA:4.303, Connector:0.075, model:1.394, lr:1.414e-06, loss:0.146loss:0.2847, grad_norm:0.3844 | |
| 2025-10-28 06:07:07 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 60104 | Local Step: 11700 // 12101, total_eta: 80 days, 17:27:46, epoch_eta:0:40:32, time: all:6.039, data:0.001, vae:0.018, IQA:4.252, Connector:0.083, model:1.281, lr:1.414e-06, loss:0.438loss:0.2553, grad_norm:0.6099 | |
| 2025-10-28 06:17:07 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 60204 | Local Step: 11800 // 12101, total_eta: 80 days, 17:07:13, epoch_eta:0:30:25, time: all:6.001, data:0.001, vae:0.018, IQA:4.232, Connector:0.069, model:1.271, lr:1.414e-06, loss:0.132loss:0.2720, grad_norm:0.3416 | |
| 2025-10-28 06:27:11 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 60304 | Local Step: 11900 // 12101, total_eta: 80 days, 16:53:07, epoch_eta:0:20:18, time: all:6.040, data:0.001, vae:0.018, IQA:4.291, Connector:0.068, model:1.267, lr:1.414e-06, loss:0.141loss:0.2746, grad_norm:0.3048 | |
| 2025-10-28 06:37:19 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 60404 | Local Step: 12000 // 12101, total_eta: 80 days, 16:46:10, epoch_eta:0:10:12, time: all:6.084, data:0.001, vae:0.019, IQA:4.376, Connector:0.067, model:1.242, lr:1.414e-06, loss:0.246loss:0.2728, grad_norm:0.9486 | |
| 2025-10-28 06:47:29 - [1m[Sana][0m - INFO - Epoch: 5 | Global Step: 60504 | Local Step: 12100 // 12101, total_eta: 80 days, 16:41:08, epoch_eta:0:00:06, time: all:6.097, data:0.001, vae:0.018, IQA:4.365, Connector:0.066, model:1.265, lr:1.414e-06, loss:0.266loss:0.2715, grad_norm:0.6559 | |
| 2025-10-28 06:47:41 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_60506_weight.pth. | |
| 2025-10-28 06:48:16 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 5 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_5_step_60506.pth. | |
| 2025-10-28 06:48:23 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 60506 | Local Step: 1 // 12101, total_eta: 99 days, 16:14:53, epoch_eta:1 day, 1:10:46, time: all:0.075, data:0.015, vae:0.000, IQA:0.041, Connector:0.001, model:0.012, lr:1.414e-06, loss:0.340loss:0.3506 | |
| 2025-10-28 06:58:43 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 60605 | Local Step: 100 // 12101, total_eta: 83 days, 9:48:45, epoch_eta:20:53:57, time: all:6.194, data:0.001, vae:0.018, IQA:4.254, Connector:0.078, model:1.371, lr:1.414e-06, loss:0.766loss:0.2725, grad_norm:0.5461 | |
| 2025-10-28 07:08:50 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 60705 | Local Step: 200 // 12101, total_eta: 82 days, 2:18:25, epoch_eta:20:24:03, time: all:6.073, data:0.001, vae:0.018, IQA:4.303, Connector:0.069, model:1.290, lr:1.414e-06, loss:0.179loss:0.2782, grad_norm:0.5767 | |
| 2025-10-28 07:18:53 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 60805 | Local Step: 300 // 12101, total_eta: 81 days, 11:11:19, epoch_eta:20:04:33, time: all:6.031, data:0.001, vae:0.018, IQA:4.254, Connector:0.071, model:1.262, lr:1.414e-06, loss:0.249loss:0.2679, grad_norm:0.4987 | |
| 2025-10-28 07:28:58 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 60905 | Local Step: 400 // 12101, total_eta: 81 days, 5:01:01, epoch_eta:19:50:41, time: all:6.049, data:0.001, vae:0.017, IQA:4.249, Connector:0.069, model:1.293, lr:1.414e-06, loss:0.171loss:0.2528, grad_norm:0.3211 | |
| 2025-10-28 07:39:24 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61005 | Local Step: 500 // 12101, total_eta: 81 days, 14:49:07, epoch_eta:19:46:32, time: all:6.262, data:0.001, vae:0.019, IQA:4.336, Connector:0.073, model:1.317, lr:1.414e-06, loss:0.249loss:0.2833, grad_norm:1.8630 | |
| 2025-10-28 07:49:32 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61105 | Local Step: 600 // 12101, total_eta: 81 days, 11:17:42, epoch_eta:19:34:18, time: all:6.074, data:0.001, vae:0.018, IQA:4.257, Connector:0.069, model:1.310, lr:1.414e-06, loss:0.094loss:0.2572, grad_norm:0.3836 | |
| 2025-10-28 07:59:37 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61205 | Local Step: 700 // 12101, total_eta: 81 days, 7:34:09, epoch_eta:19:21:58, time: all:6.048, data:0.001, vae:0.018, IQA:4.289, Connector:0.069, model:1.277, lr:1.414e-06, loss:0.009loss:0.2764, grad_norm:0.3695 | |
| 2025-10-28 08:09:47 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61305 | Local Step: 800 // 12101, total_eta: 81 days, 7:03:53, epoch_eta:19:11:35, time: all:6.107, data:0.001, vae:0.018, IQA:4.314, Connector:0.069, model:1.277, lr:1.414e-06, loss:0.688loss:0.2780, grad_norm:0.5820 | |
| 2025-10-28 08:19:55 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61405 | Local Step: 900 // 12101, total_eta: 81 days, 5:46:12, epoch_eta:19:00:44, time: all:6.082, data:0.001, vae:0.019, IQA:4.345, Connector:0.071, model:1.262, lr:1.414e-06, loss:0.137loss:0.2712, grad_norm:0.2763 | |
| 2025-10-28 08:29:58 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61505 | Local Step: 1000 // 12101, total_eta: 81 days, 2:57:10, epoch_eta:18:49:01, time: all:6.028, data:0.001, vae:0.018, IQA:4.350, Connector:0.066, model:1.223, lr:1.414e-06, loss:0.303loss:0.2902, grad_norm:0.5070 | |
| 2025-10-28 08:39:59 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61605 | Local Step: 1100 // 12101, total_eta: 81 days, 0:08:43, epoch_eta:18:37:20, time: all:6.011, data:0.001, vae:0.019, IQA:4.250, Connector:0.073, model:1.284, lr:1.414e-06, loss:0.008loss:0.2588, grad_norm:0.4154 | |
| 2025-10-28 08:49:59 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61705 | Local Step: 1200 // 12101, total_eta: 80 days, 21:25:32, epoch_eta:18:25:43, time: all:5.998, data:0.001, vae:0.018, IQA:4.337, Connector:0.066, model:1.204, lr:1.414e-06, loss:0.562loss:0.2721, grad_norm:0.6729 | |
| 2025-10-28 09:00:00 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61805 | Local Step: 1300 // 12101, total_eta: 80 days, 19:16:37, epoch_eta:18:14:27, time: all:6.005, data:0.001, vae:0.018, IQA:4.276, Connector:0.069, model:1.270, lr:1.414e-06, loss:0.805loss:0.2700, grad_norm:0.4401 | |
| 2025-10-28 09:10:12 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 61905 | Local Step: 1400 // 12101, total_eta: 80 days, 20:03:12, epoch_eta:18:04:51, time: all:6.121, data:0.001, vae:0.018, IQA:4.387, Connector:0.069, model:1.269, lr:1.414e-06, loss:0.186loss:0.2524, grad_norm:0.8836 | |
| 2025-10-28 09:20:32 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62005 | Local Step: 1500 // 12101, total_eta: 80 days, 22:31:26, epoch_eta:17:56:11, time: all:6.207, data:0.001, vae:0.019, IQA:4.264, Connector:0.081, model:1.375, lr:1.414e-06, loss:0.224loss:0.2664, grad_norm:0.6880 | |
| 2025-10-28 09:30:33 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62105 | Local Step: 1600 // 12101, total_eta: 80 days, 20:38:00, epoch_eta:17:45:05, time: all:6.005, data:0.001, vae:0.018, IQA:4.298, Connector:0.069, model:1.238, lr:1.414e-06, loss:0.172loss:0.2698, grad_norm:0.4821 | |
| 2025-10-28 09:40:34 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62205 | Local Step: 1700 // 12101, total_eta: 80 days, 19:00:08, epoch_eta:17:34:09, time: all:6.008, data:0.001, vae:0.018, IQA:4.282, Connector:0.067, model:1.273, lr:1.414e-06, loss:0.461loss:0.2596, grad_norm:0.6190 | |
| 2025-10-28 09:50:38 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62305 | Local Step: 1800 // 12101, total_eta: 80 days, 18:10:42, epoch_eta:17:23:39, time: all:6.044, data:0.001, vae:0.018, IQA:4.255, Connector:0.071, model:1.312, lr:1.414e-06, loss:0.289loss:0.2527, grad_norm:0.9830 | |
| 2025-10-28 10:00:43 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62405 | Local Step: 1900 // 12101, total_eta: 80 days, 17:32:25, epoch_eta:17:13:16, time: all:6.051, data:0.001, vae:0.018, IQA:4.333, Connector:0.069, model:1.253, lr:1.414e-06, loss:0.208loss:0.2701, grad_norm:0.3626 | |
| 2025-10-28 10:10:46 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62505 | Local Step: 2000 // 12101, total_eta: 80 days, 16:37:41, epoch_eta:17:02:45, time: all:6.031, data:0.001, vae:0.018, IQA:4.289, Connector:0.070, model:1.264, lr:1.414e-06, loss:0.906loss:0.2778, grad_norm:0.4771 | |
| 2025-10-28 10:20:48 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62605 | Local Step: 2100 // 12101, total_eta: 80 days, 15:34:13, epoch_eta:16:52:10, time: all:6.017, data:0.001, vae:0.018, IQA:4.262, Connector:0.069, model:1.287, lr:1.414e-06, loss:0.844loss:0.2645, grad_norm:0.3874 | |
| 2025-10-28 10:30:51 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62705 | Local Step: 2200 // 12101, total_eta: 80 days, 14:45:20, epoch_eta:16:41:42, time: all:6.028, data:0.001, vae:0.019, IQA:4.268, Connector:0.069, model:1.297, lr:1.414e-06, loss:0.357loss:0.2743, grad_norm:0.5035 | |
| 2025-10-28 10:40:58 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62805 | Local Step: 2300 // 12101, total_eta: 80 days, 14:34:20, epoch_eta:16:31:35, time: all:6.069, data:0.001, vae:0.018, IQA:4.309, Connector:0.068, model:1.293, lr:1.414e-06, loss:0.367loss:0.2530, grad_norm:0.3475 | |
| 2025-10-28 10:50:59 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 62905 | Local Step: 2400 // 12101, total_eta: 80 days, 13:40:31, epoch_eta:16:21:06, time: all:6.015, data:0.001, vae:0.018, IQA:4.260, Connector:0.068, model:1.292, lr:1.414e-06, loss:0.473loss:0.2712, grad_norm:0.7734 | |
| 2025-10-28 11:01:24 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63005 | Local Step: 2500 // 12101, total_eta: 80 days, 15:50:38, epoch_eta:16:12:09, time: all:6.251, data:0.001, vae:0.019, IQA:4.372, Connector:0.075, model:1.325, lr:1.414e-06, loss:0.172loss:0.2714, grad_norm:0.3589 | |
| 2025-10-28 11:11:24 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63105 | Local Step: 2600 // 12101, total_eta: 80 days, 14:44:12, epoch_eta:16:01:34, time: all:5.999, data:0.001, vae:0.018, IQA:4.277, Connector:0.069, model:1.252, lr:1.414e-06, loss:0.135loss:0.2831, grad_norm:0.3503 | |
| 2025-10-28 11:21:24 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63205 | Local Step: 2700 // 12101, total_eta: 80 days, 13:41:18, epoch_eta:15:51:01, time: all:5.998, data:0.001, vae:0.018, IQA:4.230, Connector:0.066, model:1.312, lr:1.414e-06, loss:0.371loss:0.2722, grad_norm:0.3877 | |
| 2025-10-28 11:31:31 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63305 | Local Step: 2800 // 12101, total_eta: 80 days, 13:31:42, epoch_eta:15:40:54, time: all:6.070, data:0.001, vae:0.018, IQA:4.300, Connector:0.067, model:1.304, lr:1.414e-06, loss:0.180loss:0.2737, grad_norm:0.5820 | |
| 2025-10-28 11:41:35 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63405 | Local Step: 2900 // 12101, total_eta: 80 days, 13:02:57, epoch_eta:15:30:38, time: all:6.041, data:0.001, vae:0.018, IQA:4.304, Connector:0.071, model:1.266, lr:1.414e-06, loss:0.773loss:0.2667, grad_norm:0.5278 | |
| 2025-10-28 11:51:33 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63505 | Local Step: 3000 // 12101, total_eta: 80 days, 11:53:58, epoch_eta:15:20:03, time: all:5.976, data:0.001, vae:0.018, IQA:4.252, Connector:0.070, model:1.261, lr:1.414e-06, loss:0.432loss:0.2801, grad_norm:0.4926 | |
| 2025-10-28 12:01:37 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63605 | Local Step: 3100 // 12101, total_eta: 80 days, 11:26:27, epoch_eta:15:09:48, time: all:6.037, data:0.001, vae:0.018, IQA:4.321, Connector:0.069, model:1.257, lr:1.414e-06, loss:0.008loss:0.2727, grad_norm:0.6001 | |
| 2025-10-28 12:11:43 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63705 | Local Step: 3200 // 12101, total_eta: 80 days, 11:18:52, epoch_eta:14:59:43, time: all:6.069, data:0.001, vae:0.018, IQA:4.279, Connector:0.069, model:1.312, lr:1.414e-06, loss:0.221loss:0.2717, grad_norm:0.5648 | |
| 2025-10-28 12:21:47 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63805 | Local Step: 3300 // 12101, total_eta: 80 days, 10:52:35, epoch_eta:14:49:29, time: all:6.037, data:0.001, vae:0.018, IQA:4.285, Connector:0.069, model:1.279, lr:1.414e-06, loss:0.373loss:0.2861, grad_norm:0.5972 | |
| 2025-10-28 12:31:53 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 63905 | Local Step: 3400 // 12101, total_eta: 80 days, 10:38:39, epoch_eta:14:39:21, time: all:6.057, data:0.001, vae:0.018, IQA:4.366, Connector:0.071, model:1.223, lr:1.414e-06, loss:0.152loss:0.2746, grad_norm:0.4694 | |
| 2025-10-28 12:42:12 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64005 | Local Step: 3500 // 12101, total_eta: 80 days, 11:36:38, epoch_eta:14:29:45, time: all:6.189, data:0.001, vae:0.019, IQA:4.249, Connector:0.079, model:1.387, lr:1.414e-06, loss:0.484loss:0.2743, grad_norm:0.4191 | |
| 2025-10-28 12:52:16 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64105 | Local Step: 3600 // 12101, total_eta: 80 days, 11:12:21, epoch_eta:14:19:32, time: all:6.041, data:0.001, vae:0.018, IQA:4.255, Connector:0.071, model:1.302, lr:1.414e-06, loss:0.453loss:0.2683, grad_norm:0.5254 | |
| 2025-10-28 13:02:19 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64205 | Local Step: 3700 // 12101, total_eta: 80 days, 10:43:59, epoch_eta:14:09:17, time: all:6.031, data:0.001, vae:0.018, IQA:4.278, Connector:0.067, model:1.292, lr:1.414e-06, loss:1.242loss:0.2678, grad_norm:0.4727 | |
| 2025-10-28 13:12:27 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64305 | Local Step: 3800 // 12101, total_eta: 80 days, 10:43:20, epoch_eta:13:59:15, time: all:6.085, data:0.001, vae:0.018, IQA:4.386, Connector:0.069, model:1.228, lr:1.414e-06, loss:0.122loss:0.2700, grad_norm:0.3677 | |
| 2025-10-28 13:22:27 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64405 | Local Step: 3900 // 12101, total_eta: 80 days, 10:00:10, epoch_eta:13:48:54, time: all:5.999, data:0.001, vae:0.018, IQA:4.299, Connector:0.068, model:1.228, lr:1.414e-06, loss:0.204loss:0.2826, grad_norm:0.4814 | |
| 2025-10-28 13:32:31 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64505 | Local Step: 4000 // 12101, total_eta: 80 days, 9:36:16, epoch_eta:13:38:42, time: all:6.036, data:0.001, vae:0.018, IQA:4.261, Connector:0.071, model:1.304, lr:1.414e-06, loss:0.189loss:0.2889, grad_norm:0.6646 | |
| 2025-10-28 13:42:36 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64605 | Local Step: 4100 // 12101, total_eta: 80 days, 9:18:35, epoch_eta:13:28:32, time: all:6.047, data:0.001, vae:0.019, IQA:4.276, Connector:0.071, model:1.300, lr:1.414e-06, loss:0.531loss:0.2735, grad_norm:1.9020 | |
| 2025-10-28 13:52:42 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64705 | Local Step: 4200 // 12101, total_eta: 80 days, 9:07:11, epoch_eta:13:18:25, time: all:6.060, data:0.001, vae:0.018, IQA:4.317, Connector:0.068, model:1.281, lr:1.414e-06, loss:0.007loss:0.2680, grad_norm:0.8354 | |
| 2025-10-28 14:02:43 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64805 | Local Step: 4300 // 12101, total_eta: 80 days, 8:35:31, epoch_eta:13:08:10, time: all:6.015, data:0.001, vae:0.018, IQA:4.271, Connector:0.068, model:1.281, lr:1.414e-06, loss:0.305loss:0.2654, grad_norm:0.5661 | |
| 2025-10-28 14:12:47 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 64905 | Local Step: 4400 // 12101, total_eta: 80 days, 8:16:26, epoch_eta:12:58:00, time: all:6.041, data:0.001, vae:0.018, IQA:4.298, Connector:0.072, model:1.232, lr:1.414e-06, loss:0.125loss:0.2781, grad_norm:0.7692 | |
| 2025-10-28 14:22:17 - [1m[Sana][0m - INFO - Saved trainable weights of epoch 6 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_6_step_65000_weight.pth. | |
| 2025-10-28 14:22:49 - [1m[Sana][0m - INFO - Saved resume checkpoint of epoch 6 to output_composite/sd35m_d2c_multi_bf16_lr2e6/checkpoints/epoch_6_step_65000.pth. | |
| 2025-10-28 14:23:40 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65005 | Local Step: 4500 // 12101, total_eta: 80 days, 11:25:10, epoch_eta:12:49:13, time: all:6.530, data:0.001, vae:0.019, IQA:4.293, Connector:0.077, model:1.328, lr:1.414e-06, loss:0.203loss:0.2680, grad_norm:0.2915 | |
| 2025-10-28 14:33:46 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65105 | Local Step: 4600 // 12101, total_eta: 80 days, 11:10:19, epoch_eta:12:39:04, time: all:6.061, data:0.001, vae:0.019, IQA:4.277, Connector:0.070, model:1.316, lr:1.414e-06, loss:0.322loss:0.2666, grad_norm:0.4346 | |
| 2025-10-28 14:43:51 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65205 | Local Step: 4700 // 12101, total_eta: 80 days, 10:49:02, epoch_eta:12:28:53, time: all:6.044, data:0.001, vae:0.018, IQA:4.256, Connector:0.071, model:1.320, lr:1.414e-06, loss:0.379loss:0.2652, grad_norm:0.4154 | |
| 2025-10-28 14:53:52 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65305 | Local Step: 4800 // 12101, total_eta: 80 days, 10:16:39, epoch_eta:12:18:37, time: all:6.015, data:0.001, vae:0.018, IQA:4.352, Connector:0.067, model:1.205, lr:1.414e-06, loss:0.295loss:0.2581, grad_norm:0.4032 | |
| 2025-10-28 15:03:54 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65405 | Local Step: 4900 // 12101, total_eta: 80 days, 9:47:01, epoch_eta:12:08:23, time: all:6.020, data:0.001, vae:0.018, IQA:4.306, Connector:0.068, model:1.234, lr:1.414e-06, loss:0.079loss:0.2544, grad_norm:0.2996 | |
| 2025-10-28 15:14:01 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65505 | Local Step: 5000 // 12101, total_eta: 80 days, 9:36:59, epoch_eta:11:58:16, time: all:6.069, data:0.001, vae:0.018, IQA:4.334, Connector:0.071, model:1.276, lr:1.414e-06, loss:0.127loss:0.2780, grad_norm:0.4275 | |
| 2025-10-28 15:24:12 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65605 | Local Step: 5100 // 12101, total_eta: 80 days, 9:40:18, epoch_eta:11:48:14, time: all:6.105, data:0.001, vae:0.019, IQA:4.272, Connector:0.069, model:1.346, lr:1.414e-06, loss:0.077loss:0.2738, grad_norm:0.4196 | |
| 2025-10-28 15:34:14 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65705 | Local Step: 5200 // 12101, total_eta: 80 days, 9:11:20, epoch_eta:11:38:00, time: all:6.018, data:0.001, vae:0.018, IQA:4.305, Connector:0.067, model:1.254, lr:1.414e-06, loss:0.281loss:0.2772, grad_norm:0.4401 | |
| 2025-10-28 15:44:18 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65805 | Local Step: 5300 // 12101, total_eta: 80 days, 8:50:48, epoch_eta:11:27:50, time: all:6.040, data:0.001, vae:0.018, IQA:4.287, Connector:0.070, model:1.285, lr:1.414e-06, loss:0.250loss:0.2874, grad_norm:0.3201 | |
| 2025-10-28 15:54:22 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 65905 | Local Step: 5400 // 12101, total_eta: 80 days, 8:30:56, epoch_eta:11:17:39, time: all:6.041, data:0.001, vae:0.018, IQA:4.298, Connector:0.069, model:1.271, lr:1.414e-06, loss:0.131loss:0.2707, grad_norm:0.4530 | |
| 2025-10-28 16:04:42 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66005 | Local Step: 5500 // 12101, total_eta: 80 days, 9:08:27, epoch_eta:11:07:49, time: all:6.205, data:0.001, vae:0.019, IQA:4.290, Connector:0.075, model:1.369, lr:1.414e-06, loss:0.135loss:0.2575, grad_norm:0.3206 | |
| 2025-10-28 16:14:46 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66105 | Local Step: 5600 // 12101, total_eta: 80 days, 8:48:00, epoch_eta:10:57:39, time: all:6.040, data:0.001, vae:0.018, IQA:4.316, Connector:0.069, model:1.255, lr:1.414e-06, loss:0.189loss:0.2780, grad_norm:0.4473 | |
| 2025-10-28 16:24:51 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66205 | Local Step: 5700 // 12101, total_eta: 80 days, 8:31:40, epoch_eta:10:47:29, time: all:6.051, data:0.001, vae:0.018, IQA:4.286, Connector:0.068, model:1.299, lr:1.413e-06, loss:0.143loss:0.2637, grad_norm:0.3651 | |
| 2025-10-28 16:34:51 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66305 | Local Step: 5800 // 12101, total_eta: 80 days, 7:57:19, epoch_eta:10:37:15, time: all:5.996, data:0.001, vae:0.018, IQA:4.257, Connector:0.068, model:1.278, lr:1.413e-06, loss:0.432loss:0.2689, grad_norm:0.3468 | |
| 2025-10-28 16:44:54 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66405 | Local Step: 5900 // 12101, total_eta: 80 days, 7:35:23, epoch_eta:10:27:04, time: all:6.032, data:0.001, vae:0.019, IQA:4.317, Connector:0.069, model:1.246, lr:1.413e-06, loss:0.130loss:0.2778, grad_norm:0.4564 | |
| 2025-10-28 16:54:56 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66505 | Local Step: 6000 // 12101, total_eta: 80 days, 7:10:01, epoch_eta:10:16:52, time: all:6.019, data:0.001, vae:0.018, IQA:4.278, Connector:0.068, model:1.279, lr:1.413e-06, loss:0.190loss:0.2685, grad_norm:0.4473 | |
| 2025-10-28 17:04:57 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66605 | Local Step: 6100 // 12101, total_eta: 80 days, 6:41:13, epoch_eta:10:06:40, time: all:6.007, data:0.001, vae:0.018, IQA:4.195, Connector:0.071, model:1.327, lr:1.413e-06, loss:0.410loss:0.2709, grad_norm:0.5236 | |
| 2025-10-28 17:14:59 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66705 | Local Step: 6200 // 12101, total_eta: 80 days, 6:17:03, epoch_eta:9:56:29, time: all:6.020, data:0.001, vae:0.018, IQA:4.291, Connector:0.068, model:1.275, lr:1.413e-06, loss:0.648loss:0.2920, grad_norm:0.5204 | |
| 2025-10-28 17:25:10 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66805 | Local Step: 6300 // 12101, total_eta: 80 days, 6:22:54, epoch_eta:9:46:27, time: all:6.118, data:0.001, vae:0.018, IQA:4.237, Connector:0.068, model:1.345, lr:1.413e-06, loss:0.395loss:0.2707, grad_norm:0.3652 | |
| 2025-10-28 17:35:13 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 66905 | Local Step: 6400 // 12101, total_eta: 80 days, 6:01:17, epoch_eta:9:36:17, time: all:6.027, data:0.001, vae:0.018, IQA:4.244, Connector:0.068, model:1.289, lr:1.413e-06, loss:0.185loss:0.2659, grad_norm:0.5034 | |
| 2025-10-28 17:45:36 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67005 | Local Step: 6500 // 12101, total_eta: 80 days, 6:39:26, epoch_eta:9:26:25, time: all:6.230, data:0.001, vae:0.019, IQA:4.347, Connector:0.078, model:1.323, lr:1.413e-06, loss:0.125loss:0.2733, grad_norm:0.9798 | |
| 2025-10-28 17:55:41 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67105 | Local Step: 6600 // 12101, total_eta: 80 days, 6:24:36, epoch_eta:9:16:17, time: all:6.051, data:0.001, vae:0.018, IQA:4.361, Connector:0.068, model:1.218, lr:1.413e-06, loss:0.010loss:0.2663, grad_norm:0.3457 | |
| 2025-10-28 18:05:44 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67205 | Local Step: 6700 // 12101, total_eta: 80 days, 6:03:12, epoch_eta:9:06:07, time: all:6.028, data:0.001, vae:0.018, IQA:4.269, Connector:0.070, model:1.281, lr:1.413e-06, loss:0.134loss:0.2669, grad_norm:0.4365 | |
| 2025-10-28 18:15:43 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67305 | Local Step: 6800 // 12101, total_eta: 80 days, 5:31:43, epoch_eta:8:55:54, time: all:5.991, data:0.001, vae:0.018, IQA:4.278, Connector:0.067, model:1.246, lr:1.413e-06, loss:0.322loss:0.2688, grad_norm:0.2923 | |
| 2025-10-28 18:25:46 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67405 | Local Step: 6900 // 12101, total_eta: 80 days, 5:10:54, epoch_eta:8:45:45, time: all:6.027, data:0.001, vae:0.019, IQA:4.298, Connector:0.071, model:1.248, lr:1.413e-06, loss:0.225loss:0.2720, grad_norm:0.4458 | |
| 2025-10-28 18:35:49 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67505 | Local Step: 7000 // 12101, total_eta: 80 days, 4:51:56, epoch_eta:8:35:36, time: all:6.033, data:0.001, vae:0.018, IQA:4.243, Connector:0.070, model:1.305, lr:1.413e-06, loss:0.054loss:0.2623, grad_norm:0.4989 | |
| 2025-10-28 18:45:51 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67605 | Local Step: 7100 // 12101, total_eta: 80 days, 4:30:05, epoch_eta:8:25:26, time: all:6.021, data:0.001, vae:0.018, IQA:4.319, Connector:0.065, model:1.220, lr:1.413e-06, loss:0.219loss:0.2647, grad_norm:0.4056 | |
| 2025-10-28 18:55:54 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67705 | Local Step: 7200 // 12101, total_eta: 80 days, 4:11:02, epoch_eta:8:15:17, time: all:6.030, data:0.001, vae:0.018, IQA:4.310, Connector:0.067, model:1.242, lr:1.413e-06, loss:0.336loss:0.2662, grad_norm:0.8542 | |
| 2025-10-28 19:05:56 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67805 | Local Step: 7300 // 12101, total_eta: 80 days, 3:48:38, epoch_eta:8:05:08, time: all:6.017, data:0.001, vae:0.019, IQA:4.255, Connector:0.069, model:1.289, lr:1.413e-06, loss:0.173loss:0.2624, grad_norm:0.3538 | |
| 2025-10-28 19:16:01 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 67905 | Local Step: 7400 // 12101, total_eta: 80 days, 3:35:47, epoch_eta:7:55:01, time: all:6.052, data:0.001, vae:0.018, IQA:4.224, Connector:0.072, model:1.336, lr:1.413e-06, loss:0.009loss:0.2618, grad_norm:0.3419 | |
| 2025-10-28 19:26:19 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68005 | Local Step: 7500 // 12101, total_eta: 80 days, 3:54:52, epoch_eta:7:45:02, time: all:6.178, data:0.001, vae:0.019, IQA:4.265, Connector:0.076, model:1.343, lr:1.413e-06, loss:0.176loss:0.2554, grad_norm:0.4614 | |
| 2025-10-28 19:36:18 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68105 | Local Step: 7600 // 12101, total_eta: 80 days, 3:27:32, epoch_eta:7:34:51, time: all:5.996, data:0.001, vae:0.018, IQA:4.183, Connector:0.072, model:1.316, lr:1.413e-06, loss:0.459loss:0.2694, grad_norm:0.4320 | |
| 2025-10-28 19:46:22 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68205 | Local Step: 7700 // 12101, total_eta: 80 days, 3:10:23, epoch_eta:7:24:43, time: all:6.035, data:0.001, vae:0.018, IQA:4.260, Connector:0.067, model:1.282, lr:1.413e-06, loss:0.359loss:0.2520, grad_norm:0.4538 | |
| 2025-10-28 19:56:28 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68305 | Local Step: 7800 // 12101, total_eta: 80 days, 3:00:21, epoch_eta:7:14:37, time: all:6.063, data:0.001, vae:0.018, IQA:4.209, Connector:0.070, model:1.330, lr:1.413e-06, loss:0.227loss:0.2603, grad_norm:0.3185 | |
| 2025-10-28 20:06:33 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68405 | Local Step: 7900 // 12101, total_eta: 80 days, 2:46:16, epoch_eta:7:04:30, time: all:6.047, data:0.001, vae:0.018, IQA:4.288, Connector:0.070, model:1.261, lr:1.413e-06, loss:0.008loss:0.2647, grad_norm:0.6328 | |
| 2025-10-28 20:16:41 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68505 | Local Step: 8000 // 12101, total_eta: 80 days, 2:40:09, epoch_eta:6:54:24, time: all:6.080, data:0.001, vae:0.017, IQA:4.321, Connector:0.071, model:1.246, lr:1.413e-06, loss:0.150loss:0.2598, grad_norm:0.4032 | |
| 2025-10-28 20:26:45 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68605 | Local Step: 8100 // 12101, total_eta: 80 days, 2:25:21, epoch_eta:6:44:17, time: all:6.043, data:0.001, vae:0.018, IQA:4.257, Connector:0.069, model:1.286, lr:1.413e-06, loss:0.402loss:0.2591, grad_norm:0.6969 | |
| 2025-10-28 20:36:53 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68705 | Local Step: 8200 // 12101, total_eta: 80 days, 2:17:54, epoch_eta:6:34:11, time: all:6.074, data:0.001, vae:0.018, IQA:4.298, Connector:0.070, model:1.273, lr:1.413e-06, loss:0.215loss:0.2561, grad_norm:0.3378 | |
| 2025-10-28 20:46:58 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68805 | Local Step: 8300 // 12101, total_eta: 80 days, 2:05:04, epoch_eta:6:24:04, time: all:6.051, data:0.001, vae:0.017, IQA:4.284, Connector:0.067, model:1.282, lr:1.413e-06, loss:0.465loss:0.2622, grad_norm:0.4704 | |
| 2025-10-28 20:57:00 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 68905 | Local Step: 8400 // 12101, total_eta: 80 days, 1:46:58, epoch_eta:6:13:57, time: all:6.028, data:0.001, vae:0.018, IQA:4.320, Connector:0.067, model:1.219, lr:1.413e-06, loss:0.396loss:0.2594, grad_norm:0.2935 | |
| 2025-10-28 21:07:23 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69005 | Local Step: 8500 // 12101, total_eta: 80 days, 2:13:57, epoch_eta:6:03:57, time: all:6.228, data:0.001, vae:0.019, IQA:4.298, Connector:0.078, model:1.345, lr:1.413e-06, loss:0.250loss:0.2624, grad_norm:0.4349 | |
| 2025-10-28 21:17:25 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69105 | Local Step: 8600 // 12101, total_eta: 80 days, 1:54:39, epoch_eta:5:53:49, time: all:6.023, data:0.001, vae:0.017, IQA:4.268, Connector:0.068, model:1.257, lr:1.413e-06, loss:0.008loss:0.2675, grad_norm:0.5884 | |
| 2025-10-28 21:27:32 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69205 | Local Step: 8700 // 12101, total_eta: 80 days, 1:44:04, epoch_eta:5:43:43, time: all:6.062, data:0.001, vae:0.017, IQA:4.305, Connector:0.072, model:1.251, lr:1.413e-06, loss:0.264loss:0.2777, grad_norm:0.3789 | |
| 2025-10-28 21:37:33 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69305 | Local Step: 8800 // 12101, total_eta: 80 days, 1:23:00, epoch_eta:5:33:34, time: all:6.013, data:0.001, vae:0.018, IQA:4.219, Connector:0.069, model:1.299, lr:1.413e-06, loss:0.189loss:0.2700, grad_norm:0.4006 | |
| 2025-10-28 21:47:41 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69405 | Local Step: 8900 // 12101, total_eta: 80 days, 1:15:27, epoch_eta:5:23:29, time: all:6.075, data:0.001, vae:0.018, IQA:4.243, Connector:0.070, model:1.341, lr:1.413e-06, loss:0.490loss:0.2718, grad_norm:0.3394 | |
| 2025-10-28 21:57:44 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69505 | Local Step: 9000 // 12101, total_eta: 80 days, 1:00:21, epoch_eta:5:13:21, time: all:6.040, data:0.001, vae:0.018, IQA:4.226, Connector:0.068, model:1.333, lr:1.413e-06, loss:0.008loss:0.2527, grad_norm:0.7140 | |
| 2025-10-28 22:07:51 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69605 | Local Step: 9100 // 12101, total_eta: 80 days, 0:50:11, epoch_eta:5:03:15, time: all:6.063, data:0.001, vae:0.017, IQA:4.380, Connector:0.067, model:1.181, lr:1.413e-06, loss:0.233loss:0.2682, grad_norm:0.6177 | |
| 2025-10-28 22:17:57 - [1m[Sana][0m - INFO - Epoch: 6 | Global Step: 69705 | Local Step: 9200 // 12101, total_eta: 80 days, 0:40:20, epoch_eta:4:53:09, time: all:6.064, data:0.001, vae:0.017, IQA:4.299, Connector:0.071, model:1.275, lr:1.413e-06, loss:0.309loss:0.2818, grad_norm:0.4084 | |