Buckets:
| Running 1 job | |
| { | |
| "type": "diffusion_trainer", | |
| "training_folder": "/app/ai-toolkit/output", | |
| "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", | |
| "device": "cuda", | |
| "trigger_word": null, | |
| "performance_log_every": 10, | |
| "network": { | |
| "type": "lora", | |
| "linear": 32, | |
| "linear_alpha": 32, | |
| "conv": 16, | |
| "conv_alpha": 16, | |
| "lokr_full_rank": true, | |
| "lokr_factor": -1, | |
| "network_kwargs": { | |
| "ignore_if_contains": [] | |
| } | |
| }, | |
| "save": { | |
| "dtype": "bf16", | |
| "save_every": 25, | |
| "max_step_saves_to_keep": 500, | |
| "save_format": "diffusers", | |
| "push_to_hub": false | |
| }, | |
| "datasets": [ | |
| { | |
| "folder_path": "/app/ai-toolkit/datasets/nsfw", | |
| "mask_path": null, | |
| "mask_min_value": 0.1, | |
| "default_caption": "", | |
| "caption_ext": "txt", | |
| "caption_dropout_rate": 0.05, | |
| "cache_latents_to_disk": false, | |
| "is_reg": false, | |
| "network_weight": 1, | |
| "resolution": [ | |
| 512, | |
| 256 | |
| ], | |
| "controls": [], | |
| "shrink_video_to_frames": true, | |
| "num_frames": 1, | |
| "flip_x": false, | |
| "flip_y": false, | |
| "num_repeats": 1 | |
| } | |
| ], | |
| "train": { | |
| "batch_size": 7, | |
| "bypass_guidance_embedding": false, | |
| "steps": 20000, | |
| "gradient_accumulation": 1, | |
| "train_unet": true, | |
| "train_text_encoder": false, | |
| "gradient_checkpointing": true, | |
| "noise_scheduler": "ddpm", | |
| "optimizer": "adamw8bit", | |
| "timestep_type": "sigmoid", | |
| "content_or_style": "balanced", | |
| "optimizer_params": { | |
| "weight_decay": 0.0001 | |
| }, | |
| "unload_text_encoder": false, | |
| "cache_text_embeddings": false, | |
| "lr": 7e-05, | |
| "ema_config": { | |
| "use_ema": false, | |
| "ema_decay": 0.99 | |
| }, | |
| "skip_first_sample": false, | |
| "force_first_sample": false, | |
| "disable_sampling": true, | |
| "dtype": "bf16", | |
| "diff_output_preservation": false, | |
| "diff_output_preservation_multiplier": 1, | |
| "diff_output_preservation_class": "person", | |
| "switch_boundary_every": 1, | |
| "loss_type": "mse", | |
| "do_differential_guidance": true, | |
| "differential_guidance_scale": 3 | |
| }, | |
| "logging": { | |
| "log_every": 1, | |
| "use_ui_logger": true | |
| }, | |
| "model": { | |
| "name_or_path": "stable-diffusion-v1-5/stable-diffusion-v1-5", | |
| "quantize": false, | |
| "qtype": "qfloat8", | |
| "quantize_te": false, | |
| "qtype_te": "qfloat8", | |
| "arch": "sd1", | |
| "low_vram": false, | |
| "model_kwargs": {}, | |
| "compile": false | |
| }, | |
| "sample": { | |
| "sampler": "ddpm", | |
| "sample_every": 50, | |
| "sample_start_step": 5000, | |
| "width": 512, | |
| "height": 512, | |
| "samples": [], | |
| "neg": "", | |
| "seed": 42, | |
| "walk_seed": true, | |
| "guidance_scale": 6, | |
| "sample_steps": 30, | |
| "num_frames": 1, | |
| "fps": 1 | |
| } | |
| } | |
| Using SQLite database at /app/ai-toolkit/aitk_db.db | |
| Job ID: "b189da96-babe-4a7c-846a-ad0aa5ac67e8" | |
| ############################################# | |
| # Running job: jam | |
| ############################################# | |
| Running 1 process | |
| Loading pipeline components...: 0%| | 0/6 [00:00<?, ?it/s] | |
| Loading weights: 0%| | 0/196 [00:00<?, ?it/s][A Loading weights: 100%|██████████| 196/196 [00:00<00:00, 10985.87it/s] | |
| Loading pipeline components...: 83%|████████▎ | 5/6 [00:00<00:00, 36.11it/s] Loading pipeline components...: 100%|██████████| 6/6 [00:00<00:00, 33.22it/s] | |
| create LoRA network. base dim (rank): 32, alpha: 32 | |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None | |
| apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16 | |
| create LoRA for Text Encoder: 0 modules. | |
| create LoRA for U-Net: 278 modules. | |
| enable LoRA for U-Net | |
| #### IMPORTANT RESUMING FROM /app/ai-toolkit/output/jam/jam_000001225.safetensors #### | |
| Loading from /app/ai-toolkit/output/jam/jam_000001225.safetensors | |
| Missing keys: [] | |
| Found step 1225 in metadata, starting from there | |
| Loading optimizer state from /app/ai-toolkit/output/jam/optimizer.pt | |
| Updating optimizer LR from params | |
| Dataset: /app/ai-toolkit/datasets/nsfw | |
| - Preprocessing image dimensions | |
| 0%| | 0/2672 [00:00<?, ?it/s] 100%|██████████| 2672/2672 [00:00<00:00, 84249.94it/s] | |
| - Found 2672 images | |
| Bucket sizes for /app/ai-toolkit/datasets/nsfw: | |
| 416x624: 1801 files | |
| 464x304: 96 files | |
| 448x256: 24 files | |
| 448x304: 133 files | |
| 576x448: 45 files | |
| 624x416: 349 files | |
| 672x384: 66 files | |
| 448x576: 78 files | |
| 448x352: 22 files | |
| 464x256: 38 files | |
| 384x672: 3 files | |
| 464x368: 5 files | |
| 592x432: 2 files | |
| 432x592: 2 files | |
| 464x560: 1 files | |
| 464x288: 4 files | |
| 16x16: 1 files | |
| 560x464: 1 files | |
| 464x464: 1 files | |
| 19 buckets made | |
| Dataset: /app/ai-toolkit/datasets/nsfw | |
| - Preprocessing image dimensions | |
| 0%| | 0/2672 [00:00<?, ?it/s] 100%|██████████| 2672/2672 [00:00<00:00, 86909.03it/s] | |
| - Found 2672 images | |
| Bucket sizes for /app/ai-toolkit/datasets/nsfw: | |
| 208x304: 1801 files | |
| 304x208: 578 files | |
| 336x192: 132 files | |
| 288x224: 69 files | |
| 224x288: 80 files | |
| 192x336: 3 files | |
| 272x240: 6 files | |
| 240x272: 1 files | |
| 16x16: 1 files | |
| 256x256: 1 files | |
| 10 buckets made | |
| Skipping first sample due to config setting | |
| jam: 6%|▌ | 1225/20000 [00:00<?, ?it/s] jam: 6%|▌ | 1225/20000 [00:00<?, ?it/s, lr: 7.0e-05 loss: 1.686e-01] jam: 6%|▌ | 1226/20000 [00:00<4:39:14, 1.12it/s, lr: 7.0e-05 loss: 1.686e-01] jam: 6%|▌ | 1226/20000 [00:01<4:39:14, 1.12it/s, lr: 7.0e-05 loss: 2.292e-01] jam: 6%|▌ | 1227/20000 [00:01<2:23:32, 2.18it/s, lr: 7.0e-05 loss: 2.292e-01] jam: 6%|▌ | 1227/20000 [00:01<2:23:32, 2.18it/s, lr: 7.0e-05 loss: 1.706e-01] jam: 6%|▌ | 1228/20000 [00:01<1:16:52, 4.07it/s, lr: 7.0e-05 loss: 1.706e-01] jam: 6%|▌ | 1228/20000 [00:01<1:16:52, 4.07it/s, lr: 7.0e-05 loss: 2.171e-01] jam: 6%|▌ | 1229/20000 [00:01<1:52:04, 2.79it/s, lr: 7.0e-05 loss: 2.171e-01] jam: 6%|▌ | 1229/20000 [00:01<1:52:04, 2.79it/s, lr: 7.0e-05 loss: 1.027e-01] jam: 6%|▌ | 1230/20000 [00:01<1:16:51, 4.07it/s, lr: 7.0e-05 loss: 1.027e-01] jam: 6%|▌ | 1230/20000 [00:02<1:16:51, 4.07it/s, lr: 7.0e-05 loss: 2.924e-01] jam: 6%|▌ | 1231/20000 [00:02<1:34:34, 3.31it/s, lr: 7.0e-05 loss: 2.924e-01] jam: 6%|▌ | 1231/20000 [00:02<1:34:34, 3.31it/s, lr: 7.0e-05 loss: 8.832e-02] jam: 6%|▌ | 1232/20000 [00:02<1:49:38, 2.85it/s, lr: 7.0e-05 loss: 8.832e-02] jam: 6%|▌ | 1232/20000 [00:03<1:49:38, 2.85it/s, lr: 7.0e-05 loss: 6.733e-02] jam: 6%|▌ | 1233/20000 [00:03<1:52:18, 2.79it/s, lr: 7.0e-05 loss: 6.733e-02] jam: 6%|▌ | 1233/20000 [00:03<1:52:18, 2.79it/s, lr: 7.0e-05 loss: 1.653e-01] jam: 6%|▌ | 1234/20000 [00:03<1:14:21, 4.21it/s, lr: 7.0e-05 loss: 1.653e-01] jam: 6%|▌ | 1234/20000 [00:03<1:14:21, 4.21it/s, lr: 7.0e-05 loss: 1.797e-01] jam: 6%|▌ | 1235/20000 [00:03<1:36:40, 3.24it/s, lr: 7.0e-05 loss: 1.797e-01] jam: 6%|▌ | 1235/20000 [00:03<1:36:40, 3.24it/s, lr: 7.0e-05 loss: 1.750e-01] jam: 6%|▌ | 1236/20000 [00:03<1:50:58, 2.82it/s, lr: 7.0e-05 loss: 1.750e-01] jam: 6%|▌ | 1236/20000 [00:04<1:50:58, 2.82it/s, lr: 7.0e-05 loss: 7.741e-02] jam: 6%|▌ | 1237/20000 [00:04<1:13:41, 4.24it/s, lr: 7.0e-05 loss: 7.741e-02] jam: 6%|▌ | 1237/20000 [00:04<1:13:41, 4.24it/s, lr: 7.0e-05 loss: 2.490e-01] jam: 6%|▌ | 1238/20000 [00:04<1:39:26, 3.14it/s, lr: 7.0e-05 loss: 2.490e-01] jam: 6%|▌ | 1238/20000 [00:04<1:39:26, 3.14it/s, lr: 7.0e-05 loss: 1.928e-01] jam: 6%|▌ | 1239/20000 [00:04<1:13:38, 4.25it/s, lr: 7.0e-05 loss: 1.928e-01] jam: 6%|▌ | 1239/20000 [00:05<1:13:38, 4.25it/s, lr: 7.0e-05 loss: 1.309e-01] jam: 6%|▌ | 1240/20000 [00:05<1:41:06, 3.09it/s, lr: 7.0e-05 loss: 1.309e-01] jam: 6%|▌ | 1240/20000 [00:05<1:41:06, 3.09it/s, lr: 7.0e-05 loss: 2.847e-01] jam: 6%|▌ | 1241/20000 [00:05<1:10:52, 4.41it/s, lr: 7.0e-05 loss: 2.847e-01] jam: 6%|▌ | 1241/20000 [00:05<1:10:52, 4.41it/s, lr: 7.0e-05 loss: 1.018e-01] jam: 6%|▌ | 1242/20000 [00:05<58:22, 5.36it/s, lr: 7.0e-05 loss: 1.018e-01] jam: 6%|▌ | 1242/20000 [00:05<58:22, 5.36it/s, lr: 7.0e-05 loss: 1.101e-01] jam: 6%|▌ | 1243/20000 [00:05<1:33:13, 3.35it/s, lr: 7.0e-05 loss: 1.101e-01] jam: 6%|▌ | 1243/20000 [00:05<1:33:13, 3.35it/s, lr: 7.0e-05 loss: 2.906e-01] jam: 6%|▌ | 1244/20000 [00:05<1:10:09, 4.46it/s, lr: 7.0e-05 loss: 2.906e-01] jam: 6%|▌ | 1244/20000 [00:06<1:10:09, 4.46it/s, lr: 7.0e-05 loss: 1.221e-01] jam: 6%|▌ | 1245/20000 [00:06<1:35:09, 3.28it/s, lr: 7.0e-05 loss: 1.221e-01] jam: 6%|▌ | 1245/20000 [00:06<1:35:09, 3.28it/s, lr: 7.0e-05 loss: 2.074e-01] jam: 6%|▌ | 1246/20000 [00:06<1:49:14, 2.86it/s, lr: 7.0e-05 loss: 2.074e-01] jam: 6%|▌ | 1246/20000 [00:06<1:49:14, 2.86it/s, lr: 7.0e-05 loss: 1.253e-01] jam: 6%|▌ | 1247/20000 [00:06<1:08:37, 4.55it/s, lr: 7.0e-05 loss: 1.253e-01] jam: 6%|▌ | 1247/20000 [00:07<1:08:37, 4.55it/s, lr: 7.0e-05 loss: 5.389e-02] jam: 6%|▌ | 1248/20000 [00:07<1:38:41, 3.17it/s, lr: 7.0e-05 loss: 5.389e-02] jam: 6%|▌ | 1248/20000 [00:07<1:38:41, 3.17it/s, lr: 7.0e-05 loss: 1.282e-01] jam: 6%|▌ | 1249/20000 [00:07<1:06:50, 4.68it/s, lr: 7.0e-05 loss: 1.282e-01] jam: 6%|▌ | 1249/20000 [00:07<1:06:50, 4.68it/s, lr: 7.0e-05 loss: 2.752e-01] jam: 6%|▋ | 1250/20000 [00:07<1:04:14, 4.86it/s, lr: 7.0e-05 loss: 2.752e-01] jam: 6%|▋ | 1250/20000 [00:08<1:04:14, 4.86it/s, lr: 7.0e-05 loss: 1.539e-01] | |
| Saving at step 1250 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001250.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 6%|▋ | 1251/20000 [00:08<1:04:14, 4.86it/s, lr: 7.0e-05 loss: 1.749e-01] jam: 6%|▋ | 1252/20000 [00:08<1:29:49, 3.48it/s, lr: 7.0e-05 loss: 1.749e-01] jam: 6%|▋ | 1252/20000 [00:09<1:29:49, 3.48it/s, lr: 7.0e-05 loss: 1.358e-01] jam: 6%|▋ | 1253/20000 [00:09<1:44:42, 2.98it/s, lr: 7.0e-05 loss: 1.358e-01] jam: 6%|▋ | 1253/20000 [00:09<1:44:42, 2.98it/s, lr: 7.0e-05 loss: 1.809e-01] jam: 6%|▋ | 1254/20000 [00:09<1:14:03, 4.22it/s, lr: 7.0e-05 loss: 1.809e-01] jam: 6%|▋ | 1254/20000 [00:09<1:14:03, 4.22it/s, lr: 7.0e-05 loss: 7.188e-02] jam: 6%|▋ | 1255/20000 [00:09<1:36:01, 3.25it/s, lr: 7.0e-05 loss: 7.188e-02] jam: 6%|▋ | 1255/20000 [00:09<1:36:01, 3.25it/s, lr: 7.0e-05 loss: 1.312e-01] jam: 6%|▋ | 1256/20000 [00:09<1:19:54, 3.91it/s, lr: 7.0e-05 loss: 1.312e-01] jam: 6%|▋ | 1256/20000 [00:10<1:19:54, 3.91it/s, lr: 7.0e-05 loss: 1.758e-01] jam: 6%|▋ | 1257/20000 [00:10<1:39:31, 3.14it/s, lr: 7.0e-05 loss: 1.758e-01] jam: 6%|▋ | 1257/20000 [00:10<1:39:31, 3.14it/s, lr: 7.0e-05 loss: 9.976e-02] jam: 6%|▋ | 1258/20000 [00:10<1:10:07, 4.45it/s, lr: 7.0e-05 loss: 9.976e-02] jam: 6%|▋ | 1258/20000 [00:10<1:10:07, 4.45it/s, lr: 7.0e-05 loss: 1.132e-01] jam: 6%|▋ | 1259/20000 [00:10<1:34:57, 3.29it/s, lr: 7.0e-05 loss: 1.132e-01] jam: 6%|▋ | 1259/20000 [00:11<1:34:57, 3.29it/s, lr: 7.0e-05 loss: 7.660e-02] jam: 6%|▋ | 1260/20000 [00:11<1:49:34, 2.85it/s, lr: 7.0e-05 loss: 7.660e-02] jam: 6%|▋ | 1260/20000 [00:11<1:49:34, 2.85it/s, lr: 7.0e-05 loss: 1.439e-01] jam: 6%|▋ | 1261/20000 [00:11<1:47:49, 2.90it/s, lr: 7.0e-05 loss: 1.439e-01] jam: 6%|▋ | 1261/20000 [00:11<1:47:49, 2.90it/s, lr: 7.0e-05 loss: 8.907e-02] jam: 6%|▋ | 1262/20000 [00:11<1:24:38, 3.69it/s, lr: 7.0e-05 loss: 8.907e-02] jam: 6%|▋ | 1262/20000 [00:12<1:24:38, 3.69it/s, lr: 7.0e-05 loss: 9.657e-02] jam: 6%|▋ | 1263/20000 [00:12<1:40:36, 3.10it/s, lr: 7.0e-05 loss: 9.657e-02] jam: 6%|▋ | 1263/20000 [00:12<1:40:36, 3.10it/s, lr: 7.0e-05 loss: 7.251e-02] jam: 6%|▋ | 1264/20000 [00:12<1:20:35, 3.87it/s, lr: 7.0e-05 loss: 7.251e-02] jam: 6%|▋ | 1264/20000 [00:12<1:20:35, 3.87it/s, lr: 7.0e-05 loss: 1.042e-01] jam: 6%|▋ | 1265/20000 [00:12<1:42:53, 3.03it/s, lr: 7.0e-05 loss: 1.042e-01] jam: 6%|▋ | 1265/20000 [00:12<1:42:53, 3.03it/s, lr: 7.0e-05 loss: 2.376e-01] jam: 6%|▋ | 1266/20000 [00:12<1:20:15, 3.89it/s, lr: 7.0e-05 loss: 2.376e-01] jam: 6%|▋ | 1266/20000 [00:13<1:20:15, 3.89it/s, lr: 7.0e-05 loss: 1.304e-01] jam: 6%|▋ | 1267/20000 [00:13<1:08:09, 4.58it/s, lr: 7.0e-05 loss: 1.304e-01] jam: 6%|▋ | 1267/20000 [00:13<1:08:09, 4.58it/s, lr: 7.0e-05 loss: 1.793e-01] jam: 6%|▋ | 1268/20000 [00:13<1:38:32, 3.17it/s, lr: 7.0e-05 loss: 1.793e-01] jam: 6%|▋ | 1268/20000 [00:13<1:38:32, 3.17it/s, lr: 7.0e-05 loss: 1.626e-01] jam: 6%|▋ | 1269/20000 [00:13<1:43:39, 3.01it/s, lr: 7.0e-05 loss: 1.626e-01] jam: 6%|▋ | 1269/20000 [00:13<1:43:39, 3.01it/s, lr: 7.0e-05 loss: 9.067e-02] jam: 6%|▋ | 1270/20000 [00:13<1:12:06, 4.33it/s, lr: 7.0e-05 loss: 9.067e-02] jam: 6%|▋ | 1270/20000 [00:14<1:12:06, 4.33it/s, lr: 7.0e-05 loss: 1.119e-01] jam: 6%|▋ | 1271/20000 [00:14<1:48:20, 2.88it/s, lr: 7.0e-05 loss: 1.119e-01] jam: 6%|▋ | 1271/20000 [00:14<1:48:20, 2.88it/s, lr: 7.0e-05 loss: 1.090e-01] jam: 6%|▋ | 1272/20000 [00:14<1:48:55, 2.87it/s, lr: 7.0e-05 loss: 1.090e-01] jam: 6%|▋ | 1272/20000 [00:15<1:48:55, 2.87it/s, lr: 7.0e-05 loss: 1.158e-01] jam: 6%|▋ | 1273/20000 [00:15<1:52:30, 2.77it/s, lr: 7.0e-05 loss: 1.158e-01] jam: 6%|▋ | 1273/20000 [00:15<1:52:30, 2.77it/s, lr: 7.0e-05 loss: 1.582e-01] jam: 6%|▋ | 1274/20000 [00:15<1:22:45, 3.77it/s, lr: 7.0e-05 loss: 1.582e-01] jam: 6%|▋ | 1274/20000 [00:15<1:22:45, 3.77it/s, lr: 7.0e-05 loss: 1.374e-01] jam: 6%|▋ | 1275/20000 [00:15<1:14:33, 4.19it/s, lr: 7.0e-05 loss: 1.374e-01] jam: 6%|▋ | 1275/20000 [00:16<1:14:33, 4.19it/s, lr: 7.0e-05 loss: 1.974e-01] | |
| Saving at step 1275 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001275.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 6%|▋ | 1276/20000 [00:16<1:14:33, 4.19it/s, lr: 7.0e-05 loss: 2.037e-01] jam: 6%|▋ | 1277/20000 [00:16<1:32:52, 3.36it/s, lr: 7.0e-05 loss: 2.037e-01] jam: 6%|▋ | 1277/20000 [00:17<1:32:52, 3.36it/s, lr: 7.0e-05 loss: 1.368e-01] jam: 6%|▋ | 1278/20000 [00:17<1:40:07, 3.12it/s, lr: 7.0e-05 loss: 1.368e-01] jam: 6%|▋ | 1278/20000 [00:17<1:40:07, 3.12it/s, lr: 7.0e-05 loss: 6.376e-02] jam: 6%|▋ | 1279/20000 [00:17<1:27:00, 3.59it/s, lr: 7.0e-05 loss: 6.376e-02] jam: 6%|▋ | 1279/20000 [00:17<1:27:00, 3.59it/s, lr: 7.0e-05 loss: 3.738e-02] jam: 6%|▋ | 1280/20000 [00:17<1:40:05, 3.12it/s, lr: 7.0e-05 loss: 3.738e-02] jam: 6%|▋ | 1280/20000 [00:17<1:40:05, 3.12it/s, lr: 7.0e-05 loss: 3.223e-02] jam: 6%|▋ | 1281/20000 [00:17<1:20:37, 3.87it/s, lr: 7.0e-05 loss: 3.223e-02] jam: 6%|▋ | 1281/20000 [00:18<1:20:37, 3.87it/s, lr: 7.0e-05 loss: 1.139e-01] jam: 6%|▋ | 1282/20000 [00:18<1:12:18, 4.31it/s, lr: 7.0e-05 loss: 1.139e-01] jam: 6%|▋ | 1282/20000 [00:18<1:12:18, 4.31it/s, lr: 7.0e-05 loss: 1.325e-01] jam: 6%|▋ | 1283/20000 [00:18<59:14, 5.27it/s, lr: 7.0e-05 loss: 1.325e-01] jam: 6%|▋ | 1283/20000 [00:18<59:14, 5.27it/s, lr: 7.0e-05 loss: 1.476e-01] jam: 6%|▋ | 1284/20000 [00:18<1:33:59, 3.32it/s, lr: 7.0e-05 loss: 1.476e-01] jam: 6%|▋ | 1284/20000 [00:18<1:33:59, 3.32it/s, lr: 7.0e-05 loss: 1.245e-01] jam: 6%|▋ | 1285/20000 [00:18<1:22:37, 3.78it/s, lr: 7.0e-05 loss: 1.245e-01] jam: 6%|▋ | 1285/20000 [00:19<1:22:37, 3.78it/s, lr: 7.0e-05 loss: 9.693e-02] jam: 6%|▋ | 1286/20000 [00:19<1:40:50, 3.09it/s, lr: 7.0e-05 loss: 9.693e-02] jam: 6%|▋ | 1286/20000 [00:19<1:40:50, 3.09it/s, lr: 7.0e-05 loss: 5.605e-02] jam: 6%|▋ | 1287/20000 [00:19<1:51:57, 2.79it/s, lr: 7.0e-05 loss: 5.605e-02] jam: 6%|▋ | 1287/20000 [00:20<1:51:57, 2.79it/s, lr: 7.0e-05 loss: 4.405e-02] jam: 6%|▋ | 1288/20000 [00:20<1:47:28, 2.90it/s, lr: 7.0e-05 loss: 4.405e-02] jam: 6%|▋ | 1288/20000 [00:20<1:47:28, 2.90it/s, lr: 7.0e-05 loss: 1.964e-01] jam: 6%|▋ | 1289/20000 [00:20<1:51:59, 2.78it/s, lr: 7.0e-05 loss: 1.964e-01] jam: 6%|▋ | 1289/20000 [00:20<1:51:59, 2.78it/s, lr: 7.0e-05 loss: 1.384e-01] jam: 6%|▋ | 1290/20000 [00:20<1:13:29, 4.24it/s, lr: 7.0e-05 loss: 1.384e-01] jam: 6%|▋ | 1290/20000 [00:20<1:13:29, 4.24it/s, lr: 7.0e-05 loss: 3.110e-01] jam: 6%|▋ | 1291/20000 [00:20<1:36:34, 3.23it/s, lr: 7.0e-05 loss: 3.110e-01] jam: 6%|▋ | 1291/20000 [00:21<1:36:34, 3.23it/s, lr: 7.0e-05 loss: 2.417e-01] jam: 6%|▋ | 1292/20000 [00:21<1:18:54, 3.95it/s, lr: 7.0e-05 loss: 2.417e-01] jam: 6%|▋ | 1292/20000 [00:21<1:18:54, 3.95it/s, lr: 7.0e-05 loss: 1.111e-01] jam: 6%|▋ | 1293/20000 [00:21<1:20:45, 3.86it/s, lr: 7.0e-05 loss: 1.111e-01] jam: 6%|▋ | 1293/20000 [00:21<1:20:45, 3.86it/s, lr: 7.0e-05 loss: 9.053e-02] jam: 6%|▋ | 1294/20000 [00:21<1:00:08, 5.18it/s, lr: 7.0e-05 loss: 9.053e-02] jam: 6%|▋ | 1294/20000 [00:21<1:00:08, 5.18it/s, lr: 7.0e-05 loss: 1.997e-01] jam: 6%|▋ | 1295/20000 [00:21<1:32:47, 3.36it/s, lr: 7.0e-05 loss: 1.997e-01] jam: 6%|▋ | 1295/20000 [00:22<1:32:47, 3.36it/s, lr: 7.0e-05 loss: 6.700e-02] jam: 6%|▋ | 1296/20000 [00:22<1:14:56, 4.16it/s, lr: 7.0e-05 loss: 6.700e-02] jam: 6%|▋ | 1296/20000 [00:22<1:14:56, 4.16it/s, lr: 7.0e-05 loss: 1.065e-01] jam: 6%|▋ | 1297/20000 [00:22<1:40:59, 3.09it/s, lr: 7.0e-05 loss: 1.065e-01] jam: 6%|▋ | 1297/20000 [00:22<1:40:59, 3.09it/s, lr: 7.0e-05 loss: 1.071e-01] jam: 6%|▋ | 1298/20000 [00:22<1:06:21, 4.70it/s, lr: 7.0e-05 loss: 1.071e-01] jam: 6%|▋ | 1298/20000 [00:22<1:06:21, 4.70it/s, lr: 7.0e-05 loss: 5.828e-02] jam: 6%|▋ | 1299/20000 [00:22<1:37:35, 3.19it/s, lr: 7.0e-05 loss: 5.828e-02] jam: 6%|▋ | 1299/20000 [00:23<1:37:35, 3.19it/s, lr: 7.0e-05 loss: 7.516e-02] jam: 6%|▋ | 1300/20000 [00:23<1:42:57, 3.03it/s, lr: 7.0e-05 loss: 7.516e-02] jam: 6%|▋ | 1300/20000 [00:23<1:42:57, 3.03it/s, lr: 7.0e-05 loss: 3.456e-01] | |
| Saving at step 1300 | |
| ======================================== | |
| Job stopped | |
| ======================================== | |
Xet Storage Details
- Size:
- 19.6 kB
- Xet hash:
- 1bc6a4c613188a18554ddd9d7502710ba3c2489d1818c155daf5aadba8451f58
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.