| Running 1 job |
| { |
| "type": "diffusion_trainer", |
| "training_folder": "/app/ai-toolkit/output", |
| "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", |
| "device": "cuda", |
| "trigger_word": null, |
| "performance_log_every": 10, |
| "network": { |
| "type": "lora", |
| "linear": 32, |
| "linear_alpha": 32, |
| "conv": 16, |
| "conv_alpha": 16, |
| "lokr_full_rank": true, |
| "lokr_factor": -1, |
| "network_kwargs": { |
| "ignore_if_contains": [] |
| } |
| }, |
| "save": { |
| "dtype": "bf16", |
| "save_every": 500, |
| "max_step_saves_to_keep": 400, |
| "save_format": "diffusers", |
| "push_to_hub": false |
| }, |
| "datasets": [ |
| { |
| "folder_path": "/app/ai-toolkit/datasets/49", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 49, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2, |
| "do_i2v": false, |
| "fps": 24 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/73", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 73, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/81", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 81, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/89", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 89, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/97", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 97, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/images", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512, |
| 768 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 1, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 1 |
| } |
| ], |
| "train": { |
| "batch_size": 1, |
| "bypass_guidance_embedding": false, |
| "steps": 25000, |
| "gradient_accumulation": 1, |
| "train_unet": true, |
| "train_text_encoder": false, |
| "gradient_checkpointing": true, |
| "noise_scheduler": "flowmatch", |
| "optimizer": "adamw8bit", |
| "timestep_type": "weighted", |
| "content_or_style": "balanced", |
| "optimizer_params": { |
| "weight_decay": 0.0001 |
| }, |
| "unload_text_encoder": false, |
| "cache_text_embeddings": true, |
| "lr": 0.0001, |
| "ema_config": { |
| "use_ema": false, |
| "ema_decay": 0.99 |
| }, |
| "skip_first_sample": false, |
| "force_first_sample": false, |
| "disable_sampling": true, |
| "dtype": "bf16", |
| "diff_output_preservation": false, |
| "diff_output_preservation_multiplier": 1, |
| "diff_output_preservation_class": "person", |
| "switch_boundary_every": 1, |
| "loss_type": "mse" |
| }, |
| "logging": { |
| "log_every": 1, |
| "use_ui_logger": true |
| }, |
| "model": { |
| "name_or_path": "Lightricks/LTX-2", |
| "quantize": true, |
| "qtype": "qfloat8", |
| "quantize_te": true, |
| "qtype_te": "uint6", |
| "arch": "ltx2", |
| "low_vram": true, |
| "model_kwargs": {}, |
| "layer_offloading": false, |
| "layer_offloading_text_encoder_percent": 1, |
| "layer_offloading_transformer_percent": 1 |
| }, |
| "sample": { |
| "sampler": "flowmatch", |
| "sample_every": 250, |
| "width": 768, |
| "height": 768, |
| "samples": [], |
| "neg": "", |
| "seed": 42, |
| "walk_seed": true, |
| "guidance_scale": 4, |
| "sample_steps": 25, |
| "num_frames": 121, |
| "fps": 24 |
| } |
| } |
| Changing sample extention to animated webp |
| Using SQLite database at /app/ai-toolkit/aitk_db.db |
| Job ID: "bf530ed1-e796-42a1-9ffa-55399fe92986" |
|
|
| ############################################# |
| # Running job: PLORAV7_LTX |
| ############################################# |
|
|
|
|
| Running 1 process |
| Loading LTX2 model |
| Loading transformer |
|
Loading checkpoint shards: 0%| | 0/8 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/8 [00:00<?, ?it/s]
Loading checkpoint shards: 12%|#2 | 1/8 [00:00<00:01, 3.98it/s]
Loading checkpoint shards: 12%|#2 | 1/8 [00:00<00:01, 3.98it/s]
Loading checkpoint shards: 25%|##5 | 2/8 [00:00<00:01, 3.02it/s]
Loading checkpoint shards: 25%|##5 | 2/8 [00:00<00:01, 3.02it/s]
Loading checkpoint shards: 38%|###7 | 3/8 [00:01<00:01, 2.59it/s]
Loading checkpoint shards: 38%|###7 | 3/8 [00:01<00:01, 2.59it/s]
Loading checkpoint shards: 50%|##### | 4/8 [00:01<00:01, 3.01it/s]
Loading checkpoint shards: 50%|##### | 4/8 [00:01<00:01, 3.01it/s]
Loading checkpoint shards: 62%|######2 | 5/8 [00:01<00:01, 2.69it/s]
Loading checkpoint shards: 62%|######2 | 5/8 [00:01<00:01, 2.69it/s]
Loading checkpoint shards: 75%|#######5 | 6/8 [00:02<00:00, 2.43it/s]
Loading checkpoint shards: 75%|#######5 | 6/8 [00:02<00:00, 2.43it/s]
Loading checkpoint shards: 88%|########7 | 7/8 [00:02<00:00, 2.59it/s]
Loading checkpoint shards: 88%|########7 | 7/8 [00:02<00:00, 2.59it/s]
Loading checkpoint shards: 100%|##########| 8/8 [00:02<00:00, 3.21it/s]
Loading checkpoint shards: 100%|##########| 8/8 [00:02<00:00, 3.21it/s]
Loading checkpoint shards: 100%|##########| 8/8 [00:02<00:00, 2.90it/s]
Loading checkpoint shards: 100%|##########| 8/8 [00:02<00:00, 2.90it/s] |
|
|
| Quantizing Transformer |
| - quantizing 48 transformer blocks |
|
0%| | 0/48 [00:00<?, ?it/s]
0%| | 0/48 [00:00<?, ?it/s]
2%|2 | 1/48 [00:00<00:32, 1.44it/s]
2%|2 | 1/48 [00:00<00:32, 1.44it/s]
4%|4 | 2/48 [00:01<00:25, 1.79it/s]
4%|4 | 2/48 [00:01<00:25, 1.79it/s]
6%|6 | 3/48 [00:01<00:23, 1.94it/s]
6%|6 | 3/48 [00:01<00:23, 1.94it/s]
8%|8 | 4/48 [00:02<00:21, 2.02it/s]
8%|8 | 4/48 [00:02<00:21, 2.02it/s]
10%|# | 5/48 [00:02<00:20, 2.07it/s]
10%|# | 5/48 [00:02<00:20, 2.07it/s]
12%|#2 | 6/48 [00:03<00:19, 2.10it/s]
12%|#2 | 6/48 [00:03<00:19, 2.10it/s]
15%|#4 | 7/48 [00:03<00:19, 2.13it/s]
15%|#4 | 7/48 [00:03<00:19, 2.13it/s]
17%|#6 | 8/48 [00:03<00:18, 2.14it/s]
17%|#6 | 8/48 [00:03<00:18, 2.14it/s]
19%|#8 | 9/48 [00:04<00:18, 2.16it/s]
19%|#8 | 9/48 [00:04<00:18, 2.16it/s]
21%|## | 10/48 [00:04<00:17, 2.18it/s]
21%|## | 10/48 [00:04<00:17, 2.18it/s]
23%|##2 | 11/48 [00:05<00:16, 2.18it/s]
23%|##2 | 11/48 [00:05<00:16, 2.18it/s]
25%|##5 | 12/48 [00:05<00:16, 2.19it/s]
25%|##5 | 12/48 [00:05<00:16, 2.19it/s]
27%|##7 | 13/48 [00:06<00:17, 1.98it/s]
27%|##7 | 13/48 [00:06<00:17, 1.98it/s]
29%|##9 | 14/48 [00:06<00:16, 2.05it/s]
29%|##9 | 14/48 [00:06<00:16, 2.05it/s]
31%|###1 | 15/48 [00:07<00:15, 2.10it/s]
31%|###1 | 15/48 [00:07<00:15, 2.10it/s]
33%|###3 | 16/48 [00:07<00:15, 2.13it/s]
33%|###3 | 16/48 [00:07<00:15, 2.13it/s]
35%|###5 | 17/48 [00:08<00:14, 2.15it/s]
35%|###5 | 17/48 [00:08<00:14, 2.15it/s]
38%|###7 | 18/48 [00:08<00:13, 2.17it/s]
38%|###7 | 18/48 [00:08<00:13, 2.17it/s]
40%|###9 | 19/48 [00:09<00:14, 1.99it/s]
40%|###9 | 19/48 [00:09<00:14, 1.99it/s]
42%|####1 | 20/48 [00:09<00:13, 2.05it/s]
42%|####1 | 20/48 [00:09<00:13, 2.05it/s]
44%|####3 | 21/48 [00:10<00:12, 2.08it/s]
44%|####3 | 21/48 [00:10<00:12, 2.08it/s]
46%|####5 | 22/48 [00:10<00:12, 2.11it/s]
46%|####5 | 22/48 [00:10<00:12, 2.11it/s]
48%|####7 | 23/48 [00:11<00:11, 2.14it/s]
48%|####7 | 23/48 [00:11<00:11, 2.14it/s]
50%|##### | 24/48 [00:11<00:11, 2.17it/s]
50%|##### | 24/48 [00:11<00:11, 2.17it/s]
52%|#####2 | 25/48 [00:12<00:11, 1.96it/s]
52%|#####2 | 25/48 [00:12<00:11, 1.96it/s]
54%|#####4 | 26/48 [00:12<00:10, 2.07it/s]
54%|#####4 | 26/48 [00:12<00:10, 2.07it/s]
56%|#####6 | 27/48 [00:12<00:09, 2.13it/s]
56%|#####6 | 27/48 [00:12<00:09, 2.13it/s]
58%|#####8 | 28/48 [00:13<00:09, 2.18it/s]
58%|#####8 | 28/48 [00:13<00:09, 2.18it/s]
60%|###### | 29/48 [00:13<00:08, 2.22it/s]
60%|###### | 29/48 [00:13<00:08, 2.22it/s]
62%|######2 | 30/48 [00:14<00:08, 2.25it/s]
62%|######2 | 30/48 [00:14<00:08, 2.25it/s]
65%|######4 | 31/48 [00:14<00:07, 2.27it/s]
65%|######4 | 31/48 [00:14<00:07, 2.27it/s]
67%|######6 | 32/48 [00:15<00:07, 2.01it/s]
67%|######6 | 32/48 [00:15<00:07, 2.01it/s]
69%|######8 | 33/48 [00:15<00:07, 2.06it/s]
69%|######8 | 33/48 [00:15<00:07, 2.06it/s]
71%|####### | 34/48 [00:16<00:06, 2.10it/s]
71%|####### | 34/48 [00:16<00:06, 2.10it/s]
73%|#######2 | 35/48 [00:16<00:06, 2.13it/s]
73%|#######2 | 35/48 [00:16<00:06, 2.13it/s]
75%|#######5 | 36/48 [00:17<00:05, 2.16it/s]
75%|#######5 | 36/48 [00:17<00:05, 2.16it/s]
77%|#######7 | 37/48 [00:17<00:05, 2.18it/s]
77%|#######7 | 37/48 [00:17<00:05, 2.18it/s]
79%|#######9 | 38/48 [00:18<00:05, 1.98it/s]
79%|#######9 | 38/48 [00:18<00:05, 1.98it/s]
81%|########1 | 39/48 [00:18<00:04, 1.93it/s]
81%|########1 | 39/48 [00:18<00:04, 1.93it/s]
83%|########3 | 40/48 [00:19<00:04, 1.86it/s]
83%|########3 | 40/48 [00:19<00:04, 1.86it/s]
85%|########5 | 41/48 [00:19<00:03, 1.82it/s]
85%|########5 | 41/48 [00:19<00:03, 1.82it/s]
88%|########7 | 42/48 [00:20<00:03, 1.78it/s]
88%|########7 | 42/48 [00:20<00:03, 1.78it/s]
90%|########9 | 43/48 [00:21<00:02, 1.73it/s]
90%|########9 | 43/48 [00:21<00:02, 1.73it/s]
92%|#########1| 44/48 [00:21<00:02, 1.71it/s]
92%|#########1| 44/48 [00:21<00:02, 1.71it/s]
94%|#########3| 45/48 [00:22<00:01, 1.52it/s]
94%|#########3| 45/48 [00:22<00:01, 1.52it/s]
96%|#########5| 46/48 [00:23<00:01, 1.55it/s]
96%|#########5| 46/48 [00:23<00:01, 1.55it/s]
98%|#########7| 47/48 [00:23<00:00, 1.56it/s]
98%|#########7| 47/48 [00:23<00:00, 1.56it/s]
100%|##########| 48/48 [00:24<00:00, 1.58it/s]
100%|##########| 48/48 [00:24<00:00, 1.58it/s]
100%|##########| 48/48 [00:24<00:00, 1.97it/s]
100%|##########| 48/48 [00:24<00:00, 1.97it/s] |
|
|
| - quantizing extras |
| Moving transformer to CPU |
| Loading text encoder |
|
Loading checkpoint shards: 0%| | 0/11 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/11 [00:00<?, ?it/s]
Loading checkpoint shards: 9%|9 | 1/11 [00:05<00:50, 5.08s/it]
Loading checkpoint shards: 9%|9 | 1/11 [00:05<00:50, 5.08s/it]
Loading checkpoint shards: 18%|#8 | 2/11 [00:10<00:48, 5.38s/it]
Loading checkpoint shards: 18%|#8 | 2/11 [00:10<00:48, 5.38s/it]
Loading checkpoint shards: 27%|##7 | 3/11 [00:14<00:37, 4.66s/it]
Loading checkpoint shards: 27%|##7 | 3/11 [00:14<00:37, 4.66s/it]
Loading checkpoint shards: 36%|###6 | 4/11 [00:17<00:28, 4.05s/it]
Loading checkpoint shards: 36%|###6 | 4/11 [00:17<00:28, 4.05s/it]
Loading checkpoint shards: 45%|####5 | 5/11 [00:21<00:23, 3.92s/it]
Loading checkpoint shards: 45%|####5 | 5/11 [00:21<00:23, 3.92s/it]
Loading checkpoint shards: 55%|#####4 | 6/11 [00:25<00:19, 3.92s/it]
Loading checkpoint shards: 55%|#####4 | 6/11 [00:25<00:19, 3.92s/it]
Loading checkpoint shards: 64%|######3 | 7/11 [00:30<00:17, 4.31s/it]
Loading checkpoint shards: 64%|######3 | 7/11 [00:30<00:17, 4.31s/it]
Loading checkpoint shards: 73%|#######2 | 8/11 [00:35<00:14, 4.69s/it]
Loading checkpoint shards: 73%|#######2 | 8/11 [00:35<00:14, 4.69s/it]
Loading checkpoint shards: 82%|########1 | 9/11 [00:40<00:09, 4.85s/it]
Loading checkpoint shards: 82%|########1 | 9/11 [00:40<00:09, 4.85s/it]
Loading checkpoint shards: 91%|######### | 10/11 [00:45<00:04, 4.83s/it]
Loading checkpoint shards: 91%|######### | 10/11 [00:45<00:04, 4.83s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:48<00:00, 4.28s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:48<00:00, 4.28s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:48<00:00, 4.44s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:48<00:00, 4.44s/it] |
|
|
| Quantizing Text Encoder |
| Loading VAEs and other components |
| Making pipe |
| Preparing Model |
| Model Loaded |
| create LoRA network. base dim (rank): 32, alpha: 32 |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None |
| apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16 |
| create LoRA for Text Encoder: 0 modules. |
| create LoRA for U-Net: 1344 modules. |
| enable LoRA for U-Net |
| Dataset: /app/ai-toolkit/datasets/49 |
| - Preprocessing video dimensions |
|
0%| | 0/140 [00:00<?, ?it/s]
0%| | 0/140 [00:00<?, ?it/s]
100%|##########| 140/140 [00:00<00:00, 27708.69it/s]
100%|##########| 140/140 [00:00<00:00, 27708.69it/s] |
|
|
| - Found 140 videos |
| Bucket sizes for /app/ai-toolkit/datasets/49: |
| 576x416: 78 files |
| 576x448: 44 files |
| 544x416: 16 files |
| 512x384: 2 files |
| 4 buckets made |
| Caching latents for /app/ai-toolkit/datasets/49 |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/140 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/140 [00:00<?, ?it/s]
Caching latents to disk: 100%|##########| 140/140 [00:00<00:00, 37923.18it/s]
Caching latents to disk: 100%|##########| 140/140 [00:00<00:00, 37923.18it/s] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/49 |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/140 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/140 [00:00<?, ?it/s]
Caching text embeddings to disk: 1%| | 1/140 [00:12<29:19, 12.66s/it]
Caching text embeddings to disk: 1%| | 1/140 [00:12<29:19, 12.66s/it]
Caching text embeddings to disk: 1%|1 | 2/140 [00:13<12:54, 5.61s/it]
Caching text embeddings to disk: 1%|1 | 2/140 [00:13<12:54, 5.61s/it]
Caching text embeddings to disk: 2%|2 | 3/140 [00:14<07:39, 3.36s/it]
Caching text embeddings to disk: 2%|2 | 3/140 [00:14<07:39, 3.36s/it]
Caching text embeddings to disk: 3%|2 | 4/140 [00:14<05:12, 2.29s/it]
Caching text embeddings to disk: 3%|2 | 4/140 [00:14<05:12, 2.29s/it]
Caching text embeddings to disk: 4%|3 | 5/140 [00:15<03:50, 1.71s/it]
Caching text embeddings to disk: 4%|3 | 5/140 [00:15<03:50, 1.71s/it]
Caching text embeddings to disk: 4%|4 | 6/140 [00:16<03:01, 1.36s/it]
Caching text embeddings to disk: 4%|4 | 6/140 [00:16<03:01, 1.36s/it]
Caching text embeddings to disk: 5%|5 | 7/140 [00:16<02:31, 1.14s/it]
Caching text embeddings to disk: 5%|5 | 7/140 [00:16<02:31, 1.14s/it]
Caching text embeddings to disk: 6%|5 | 8/140 [00:17<02:10, 1.01it/s]
Caching text embeddings to disk: 6%|5 | 8/140 [00:17<02:10, 1.01it/s]
Caching text embeddings to disk: 6%|6 | 9/140 [00:18<01:57, 1.12it/s]
Caching text embeddings to disk: 6%|6 | 9/140 [00:18<01:57, 1.12it/s]
Caching text embeddings to disk: 7%|7 | 10/140 [00:18<01:47, 1.21it/s]
Caching text embeddings to disk: 7%|7 | 10/140 [00:18<01:47, 1.21it/s]
Caching text embeddings to disk: 8%|7 | 11/140 [00:19<01:40, 1.28it/s]
Caching text embeddings to disk: 8%|7 | 11/140 [00:19<01:40, 1.28it/s]
Caching text embeddings to disk: 9%|8 | 12/140 [00:20<01:35, 1.35it/s]
Caching text embeddings to disk: 9%|8 | 12/140 [00:20<01:35, 1.35it/s]
Caching text embeddings to disk: 9%|9 | 13/140 [00:20<01:31, 1.39it/s]
Caching text embeddings to disk: 9%|9 | 13/140 [00:20<01:31, 1.39it/s]
Caching text embeddings to disk: 10%|# | 14/140 [00:21<01:28, 1.42it/s]
Caching text embeddings to disk: 10%|# | 14/140 [00:21<01:28, 1.42it/s]
Caching text embeddings to disk: 11%|# | 15/140 [00:22<01:27, 1.43it/s]
Caching text embeddings to disk: 11%|# | 15/140 [00:22<01:27, 1.43it/s]
Caching text embeddings to disk: 11%|#1 | 16/140 [00:22<01:26, 1.44it/s]
Caching text embeddings to disk: 11%|#1 | 16/140 [00:22<01:26, 1.44it/s]
Caching text embeddings to disk: 12%|#2 | 17/140 [00:23<01:25, 1.44it/s]
Caching text embeddings to disk: 12%|#2 | 17/140 [00:23<01:25, 1.44it/s]
Caching text embeddings to disk: 13%|#2 | 18/140 [00:24<01:24, 1.44it/s]
Caching text embeddings to disk: 13%|#2 | 18/140 [00:24<01:24, 1.44it/s]
Caching text embeddings to disk: 14%|#3 | 19/140 [00:24<01:22, 1.46it/s]
Caching text embeddings to disk: 14%|#3 | 19/140 [00:24<01:22, 1.46it/s]
Caching text embeddings to disk: 14%|#4 | 20/140 [00:25<01:21, 1.47it/s]
Caching text embeddings to disk: 14%|#4 | 20/140 [00:25<01:21, 1.47it/s]
Caching text embeddings to disk: 15%|#5 | 21/140 [00:26<01:20, 1.48it/s]
Caching text embeddings to disk: 15%|#5 | 21/140 [00:26<01:20, 1.48it/s]
Caching text embeddings to disk: 16%|#5 | 22/140 [00:26<01:19, 1.48it/s]
Caching text embeddings to disk: 16%|#5 | 22/140 [00:26<01:19, 1.48it/s]
Caching text embeddings to disk: 16%|#6 | 23/140 [00:27<01:19, 1.47it/s]
Caching text embeddings to disk: 16%|#6 | 23/140 [00:27<01:19, 1.47it/s]
Caching text embeddings to disk: 17%|#7 | 24/140 [00:28<01:18, 1.47it/s]
Caching text embeddings to disk: 17%|#7 | 24/140 [00:28<01:18, 1.47it/s]
Caching text embeddings to disk: 18%|#7 | 25/140 [00:28<01:17, 1.48it/s]
Caching text embeddings to disk: 18%|#7 | 25/140 [00:28<01:17, 1.48it/s]
Caching text embeddings to disk: 19%|#8 | 26/140 [00:29<01:17, 1.47it/s]
Caching text embeddings to disk: 19%|#8 | 26/140 [00:29<01:17, 1.47it/s]
Caching text embeddings to disk: 19%|#9 | 27/140 [00:30<01:16, 1.49it/s]
Caching text embeddings to disk: 19%|#9 | 27/140 [00:30<01:16, 1.49it/s]
Caching text embeddings to disk: 20%|## | 28/140 [00:30<01:15, 1.48it/s]
Caching text embeddings to disk: 20%|## | 28/140 [00:30<01:15, 1.48it/s]
Caching text embeddings to disk: 21%|## | 29/140 [00:31<01:15, 1.47it/s]
Caching text embeddings to disk: 21%|## | 29/140 [00:31<01:15, 1.47it/s]
Caching text embeddings to disk: 21%|##1 | 30/140 [00:32<01:14, 1.49it/s]
Caching text embeddings to disk: 21%|##1 | 30/140 [00:32<01:14, 1.49it/s]
Caching text embeddings to disk: 22%|##2 | 31/140 [00:32<01:13, 1.48it/s]
Caching text embeddings to disk: 22%|##2 | 31/140 [00:32<01:13, 1.48it/s]
Caching text embeddings to disk: 23%|##2 | 32/140 [00:33<01:12, 1.49it/s]
Caching text embeddings to disk: 23%|##2 | 32/140 [00:33<01:12, 1.49it/s]
Caching text embeddings to disk: 24%|##3 | 33/140 [00:34<01:11, 1.50it/s]
Caching text embeddings to disk: 24%|##3 | 33/140 [00:34<01:11, 1.50it/s]
Caching text embeddings to disk: 24%|##4 | 34/140 [00:34<01:11, 1.48it/s]
Caching text embeddings to disk: 24%|##4 | 34/140 [00:34<01:11, 1.48it/s]
Caching text embeddings to disk: 25%|##5 | 35/140 [00:35<01:10, 1.48it/s]
Caching text embeddings to disk: 25%|##5 | 35/140 [00:35<01:10, 1.48it/s]
Caching text embeddings to disk: 26%|##5 | 36/140 [00:36<01:09, 1.50it/s]
Caching text embeddings to disk: 26%|##5 | 36/140 [00:36<01:09, 1.50it/s]
Caching text embeddings to disk: 26%|##6 | 37/140 [00:36<01:09, 1.49it/s]
Caching text embeddings to disk: 26%|##6 | 37/140 [00:36<01:09, 1.49it/s]
Caching text embeddings to disk: 27%|##7 | 38/140 [00:37<01:08, 1.49it/s]
Caching text embeddings to disk: 27%|##7 | 38/140 [00:37<01:08, 1.49it/s]
Caching text embeddings to disk: 28%|##7 | 39/140 [00:38<01:07, 1.49it/s]
Caching text embeddings to disk: 28%|##7 | 39/140 [00:38<01:07, 1.49it/s]
Caching text embeddings to disk: 29%|##8 | 40/140 [00:38<01:07, 1.49it/s]
Caching text embeddings to disk: 29%|##8 | 40/140 [00:38<01:07, 1.49it/s]
Caching text embeddings to disk: 29%|##9 | 41/140 [00:39<01:07, 1.47it/s]
Caching text embeddings to disk: 29%|##9 | 41/140 [00:39<01:07, 1.47it/s]
Caching text embeddings to disk: 30%|### | 42/140 [00:40<01:06, 1.48it/s]
Caching text embeddings to disk: 30%|### | 42/140 [00:40<01:06, 1.48it/s]
Caching text embeddings to disk: 31%|### | 43/140 [00:41<01:05, 1.47it/s]
Caching text embeddings to disk: 31%|### | 43/140 [00:41<01:05, 1.47it/s]
Caching text embeddings to disk: 31%|###1 | 44/140 [00:41<01:05, 1.47it/s]
Caching text embeddings to disk: 31%|###1 | 44/140 [00:41<01:05, 1.47it/s]
Caching text embeddings to disk: 32%|###2 | 45/140 [00:42<01:04, 1.48it/s]
Caching text embeddings to disk: 32%|###2 | 45/140 [00:42<01:04, 1.48it/s]
Caching text embeddings to disk: 33%|###2 | 46/140 [00:43<01:03, 1.47it/s]
Caching text embeddings to disk: 33%|###2 | 46/140 [00:43<01:03, 1.47it/s]
Caching text embeddings to disk: 34%|###3 | 47/140 [00:43<01:03, 1.47it/s]
Caching text embeddings to disk: 34%|###3 | 47/140 [00:43<01:03, 1.47it/s]
Caching text embeddings to disk: 34%|###4 | 48/140 [00:44<01:02, 1.47it/s]
Caching text embeddings to disk: 34%|###4 | 48/140 [00:44<01:02, 1.47it/s]
Caching text embeddings to disk: 35%|###5 | 49/140 [00:45<01:01, 1.49it/s]
Caching text embeddings to disk: 35%|###5 | 49/140 [00:45<01:01, 1.49it/s]
Caching text embeddings to disk: 36%|###5 | 50/140 [00:45<01:00, 1.49it/s]
Caching text embeddings to disk: 36%|###5 | 50/140 [00:45<01:00, 1.49it/s]
Caching text embeddings to disk: 36%|###6 | 51/140 [00:46<00:59, 1.49it/s]
Caching text embeddings to disk: 36%|###6 | 51/140 [00:46<00:59, 1.49it/s]
Caching text embeddings to disk: 37%|###7 | 52/140 [00:47<00:58, 1.49it/s]
Caching text embeddings to disk: 37%|###7 | 52/140 [00:47<00:58, 1.49it/s]
Caching text embeddings to disk: 38%|###7 | 53/140 [00:47<00:58, 1.50it/s]
Caching text embeddings to disk: 38%|###7 | 53/140 [00:47<00:58, 1.50it/s]
Caching text embeddings to disk: 39%|###8 | 54/140 [00:48<00:57, 1.49it/s]
Caching text embeddings to disk: 39%|###8 | 54/140 [00:48<00:57, 1.49it/s]
Caching text embeddings to disk: 39%|###9 | 55/140 [00:49<00:57, 1.48it/s]
Caching text embeddings to disk: 39%|###9 | 55/140 [00:49<00:57, 1.48it/s]
Caching text embeddings to disk: 40%|#### | 56/140 [00:49<00:58, 1.45it/s]
Caching text embeddings to disk: 40%|#### | 56/140 [00:49<00:58, 1.45it/s]
Caching text embeddings to disk: 41%|#### | 57/140 [00:50<00:56, 1.46it/s]
Caching text embeddings to disk: 41%|#### | 57/140 [00:50<00:56, 1.46it/s]
Caching text embeddings to disk: 41%|####1 | 58/140 [00:51<00:55, 1.47it/s]
Caching text embeddings to disk: 41%|####1 | 58/140 [00:51<00:55, 1.47it/s]
Caching text embeddings to disk: 42%|####2 | 59/140 [00:51<00:55, 1.46it/s]
Caching text embeddings to disk: 42%|####2 | 59/140 [00:51<00:55, 1.46it/s]
Caching text embeddings to disk: 43%|####2 | 60/140 [00:52<00:54, 1.46it/s]
Caching text embeddings to disk: 43%|####2 | 60/140 [00:52<00:54, 1.46it/s]
Caching text embeddings to disk: 44%|####3 | 61/140 [00:53<00:53, 1.48it/s]
Caching text embeddings to disk: 44%|####3 | 61/140 [00:53<00:53, 1.48it/s]
Caching text embeddings to disk: 44%|####4 | 62/140 [00:53<00:52, 1.49it/s]
Caching text embeddings to disk: 44%|####4 | 62/140 [00:53<00:52, 1.49it/s]
Caching text embeddings to disk: 45%|####5 | 63/140 [00:54<00:51, 1.49it/s]
Caching text embeddings to disk: 45%|####5 | 63/140 [00:54<00:51, 1.49it/s]
Caching text embeddings to disk: 46%|####5 | 64/140 [00:55<00:51, 1.47it/s]
Caching text embeddings to disk: 46%|####5 | 64/140 [00:55<00:51, 1.47it/s]
Caching text embeddings to disk: 46%|####6 | 65/140 [00:55<00:51, 1.46it/s]
Caching text embeddings to disk: 46%|####6 | 65/140 [00:55<00:51, 1.46it/s]
Caching text embeddings to disk: 47%|####7 | 66/140 [00:56<00:50, 1.48it/s]
Caching text embeddings to disk: 47%|####7 | 66/140 [00:56<00:50, 1.48it/s]
Caching text embeddings to disk: 48%|####7 | 67/140 [00:57<00:48, 1.49it/s]
Caching text embeddings to disk: 48%|####7 | 67/140 [00:57<00:48, 1.49it/s]
Caching text embeddings to disk: 49%|####8 | 68/140 [00:57<00:48, 1.47it/s]
Caching text embeddings to disk: 49%|####8 | 68/140 [00:57<00:48, 1.47it/s]
Caching text embeddings to disk: 49%|####9 | 69/140 [00:58<00:48, 1.47it/s]
Caching text embeddings to disk: 49%|####9 | 69/140 [00:58<00:48, 1.47it/s]
Caching text embeddings to disk: 50%|##### | 70/140 [00:59<00:47, 1.48it/s]
Caching text embeddings to disk: 50%|##### | 70/140 [00:59<00:47, 1.48it/s]
Caching text embeddings to disk: 100%|##########| 140/140 [00:59<00:00, 2.36it/s]
Caching text embeddings to disk: 100%|##########| 140/140 [00:59<00:00, 2.36it/s] |
|
|
| Dataset: /app/ai-toolkit/datasets/73 |
| - Preprocessing video dimensions |
|
0%| | 0/122 [00:00<?, ?it/s]
0%| | 0/122 [00:00<?, ?it/s]
16%|#5 | 19/122 [00:00<00:00, 187.83it/s]
16%|#5 | 19/122 [00:00<00:00, 187.83it/s]
31%|###1 | 38/122 [00:00<00:00, 164.03it/s]
31%|###1 | 38/122 [00:00<00:00, 164.03it/s]
45%|####5 | 55/122 [00:00<00:00, 161.12it/s]
45%|####5 | 55/122 [00:00<00:00, 161.12it/s]
100%|##########| 122/122 [00:00<00:00, 327.02it/s]
100%|##########| 122/122 [00:00<00:00, 327.02it/s] |
|
|
| - Found 122 videos |
| Bucket sizes for /app/ai-toolkit/datasets/73: |
| 576x416: 104 files |
| 576x448: 6 files |
| 672x384: 12 files |
| 3 buckets made |
| Caching latents for /app/ai-toolkit/datasets/73 |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/122 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/122 [00:00<?, ?it/s]
Caching latents to disk: 1%| | 1/122 [00:06<12:51, 6.38s/it]
Caching latents to disk: 1%| | 1/122 [00:06<12:51, 6.38s/it]
Caching latents to disk: 2%|1 | 2/122 [00:12<12:08, 6.07s/it]
Caching latents to disk: 2%|1 | 2/122 [00:12<12:08, 6.07s/it]
Caching latents to disk: 2%|2 | 3/122 [00:18<11:58, 6.04s/it]
Caching latents to disk: 2%|2 | 3/122 [00:18<11:58, 6.04s/it]
Caching latents to disk: 3%|3 | 4/122 [00:23<11:28, 5.83s/it]
Caching latents to disk: 3%|3 | 4/122 [00:23<11:28, 5.83s/it]
Caching latents to disk: 4%|4 | 5/122 [00:29<11:16, 5.79s/it]
Caching latents to disk: 4%|4 | 5/122 [00:29<11:16, 5.79s/it]
Caching latents to disk: 5%|4 | 6/122 [00:34<10:54, 5.64s/it]
Caching latents to disk: 5%|4 | 6/122 [00:34<10:54, 5.64s/it]
Caching latents to disk: 6%|5 | 7/122 [00:40<10:48, 5.64s/it]
Caching latents to disk: 6%|5 | 7/122 [00:40<10:48, 5.64s/it]
Caching latents to disk: 7%|6 | 8/122 [00:46<10:41, 5.63s/it]
Caching latents to disk: 7%|6 | 8/122 [00:46<10:41, 5.63s/it]
Caching latents to disk: 7%|7 | 9/122 [00:51<10:41, 5.68s/it]
Caching latents to disk: 7%|7 | 9/122 [00:51<10:41, 5.68s/it]
Caching latents to disk: 8%|8 | 10/122 [00:57<10:36, 5.68s/it]
Caching latents to disk: 8%|8 | 10/122 [00:57<10:36, 5.68s/it]
Caching latents to disk: 9%|9 | 11/122 [01:03<10:28, 5.66s/it]
Caching latents to disk: 9%|9 | 11/122 [01:03<10:28, 5.66s/it]
Caching latents to disk: 10%|9 | 12/122 [01:09<10:30, 5.73s/it]
Caching latents to disk: 10%|9 | 12/122 [01:09<10:30, 5.73s/it]
Caching latents to disk: 11%|# | 13/122 [01:14<10:20, 5.69s/it]
Caching latents to disk: 11%|# | 13/122 [01:14<10:20, 5.69s/it]
Caching latents to disk: 11%|#1 | 14/122 [01:20<10:04, 5.60s/it]
Caching latents to disk: 11%|#1 | 14/122 [01:20<10:04, 5.60s/it]
Caching latents to disk: 12%|#2 | 15/122 [01:26<10:15, 5.75s/it]
Caching latents to disk: 12%|#2 | 15/122 [01:26<10:15, 5.75s/it]
Caching latents to disk: 13%|#3 | 16/122 [01:32<10:20, 5.85s/it]
Caching latents to disk: 13%|#3 | 16/122 [01:32<10:20, 5.85s/it]
Caching latents to disk: 14%|#3 | 17/122 [01:38<10:26, 5.96s/it]
Caching latents to disk: 14%|#3 | 17/122 [01:38<10:26, 5.96s/it]
Caching latents to disk: 15%|#4 | 18/122 [01:43<10:02, 5.79s/it]
Caching latents to disk: 15%|#4 | 18/122 [01:43<10:02, 5.79s/it]
Caching latents to disk: 16%|#5 | 19/122 [01:49<09:53, 5.76s/it]
Caching latents to disk: 16%|#5 | 19/122 [01:49<09:53, 5.76s/it]
Caching latents to disk: 16%|#6 | 20/122 [01:55<09:49, 5.78s/it]
Caching latents to disk: 16%|#6 | 20/122 [01:55<09:49, 5.78s/it]
Caching latents to disk: 17%|#7 | 21/122 [02:01<09:56, 5.91s/it]
Caching latents to disk: 17%|#7 | 21/122 [02:01<09:56, 5.91s/it]
Caching latents to disk: 18%|#8 | 22/122 [02:07<09:40, 5.81s/it]
Caching latents to disk: 18%|#8 | 22/122 [02:07<09:40, 5.81s/it]
Caching latents to disk: 19%|#8 | 23/122 [02:13<09:47, 5.93s/it]
Caching latents to disk: 19%|#8 | 23/122 [02:13<09:47, 5.93s/it]
Caching latents to disk: 20%|#9 | 24/122 [02:19<09:36, 5.88s/it]
Caching latents to disk: 20%|#9 | 24/122 [02:19<09:36, 5.88s/it]
Caching latents to disk: 20%|## | 25/122 [02:26<10:20, 6.39s/it]
Caching latents to disk: 20%|## | 25/122 [02:26<10:20, 6.39s/it]
Caching latents to disk: 21%|##1 | 26/122 [02:32<10:00, 6.26s/it]
Caching latents to disk: 21%|##1 | 26/122 [02:32<10:00, 6.26s/it]
Caching latents to disk: 22%|##2 | 27/122 [02:40<10:35, 6.69s/it]
Caching latents to disk: 22%|##2 | 27/122 [02:40<10:35, 6.69s/it]
Caching latents to disk: 23%|##2 | 28/122 [02:46<10:26, 6.67s/it]
Caching latents to disk: 23%|##2 | 28/122 [02:46<10:26, 6.67s/it]
Caching latents to disk: 24%|##3 | 29/122 [02:54<10:45, 6.94s/it]
Caching latents to disk: 24%|##3 | 29/122 [02:54<10:45, 6.94s/it]
Caching latents to disk: 25%|##4 | 30/122 [03:01<10:44, 7.01s/it]
Caching latents to disk: 25%|##4 | 30/122 [03:01<10:44, 7.01s/it]
Caching latents to disk: 25%|##5 | 31/122 [03:09<11:00, 7.26s/it]
Caching latents to disk: 25%|##5 | 31/122 [03:09<11:00, 7.26s/it]
Caching latents to disk: 26%|##6 | 32/122 [03:17<11:04, 7.38s/it]
Caching latents to disk: 26%|##6 | 32/122 [03:17<11:04, 7.38s/it]
Caching latents to disk: 27%|##7 | 33/122 [03:24<10:46, 7.26s/it]
Caching latents to disk: 27%|##7 | 33/122 [03:24<10:46, 7.26s/it]
Caching latents to disk: 28%|##7 | 34/122 [03:32<10:56, 7.46s/it]
Caching latents to disk: 28%|##7 | 34/122 [03:32<10:56, 7.46s/it]
Caching latents to disk: 29%|##8 | 35/122 [03:39<10:55, 7.53s/it]
Caching latents to disk: 29%|##8 | 35/122 [03:39<10:55, 7.53s/it]
Caching latents to disk: 30%|##9 | 36/122 [03:47<10:54, 7.61s/it]
Caching latents to disk: 30%|##9 | 36/122 [03:47<10:54, 7.61s/it]
Caching latents to disk: 30%|### | 37/122 [03:55<10:47, 7.61s/it]
Caching latents to disk: 30%|### | 37/122 [03:55<10:47, 7.61s/it]
Caching latents to disk: 31%|###1 | 38/122 [04:02<10:34, 7.55s/it]
Caching latents to disk: 31%|###1 | 38/122 [04:02<10:34, 7.55s/it]
Caching latents to disk: 32%|###1 | 39/122 [04:10<10:27, 7.56s/it]
Caching latents to disk: 32%|###1 | 39/122 [04:10<10:27, 7.56s/it]
Caching latents to disk: 33%|###2 | 40/122 [04:16<09:57, 7.29s/it]
Caching latents to disk: 33%|###2 | 40/122 [04:16<09:57, 7.29s/it]
Caching latents to disk: 34%|###3 | 41/122 [04:23<09:43, 7.20s/it]
Caching latents to disk: 34%|###3 | 41/122 [04:23<09:43, 7.20s/it]
Caching latents to disk: 34%|###4 | 42/122 [04:30<09:32, 7.16s/it]
Caching latents to disk: 34%|###4 | 42/122 [04:30<09:32, 7.16s/it]
Caching latents to disk: 35%|###5 | 43/122 [04:37<09:21, 7.11s/it]
Caching latents to disk: 35%|###5 | 43/122 [04:37<09:21, 7.11s/it]
Caching latents to disk: 36%|###6 | 44/122 [04:45<09:28, 7.29s/it]
Caching latents to disk: 36%|###6 | 44/122 [04:45<09:28, 7.29s/it]
Caching latents to disk: 37%|###6 | 45/122 [04:53<09:42, 7.56s/it]
Caching latents to disk: 37%|###6 | 45/122 [04:53<09:42, 7.56s/it]
Caching latents to disk: 38%|###7 | 46/122 [05:00<09:19, 7.37s/it]
Caching latents to disk: 38%|###7 | 46/122 [05:00<09:19, 7.37s/it]
Caching latents to disk: 39%|###8 | 47/122 [05:08<09:21, 7.49s/it]
Caching latents to disk: 39%|###8 | 47/122 [05:08<09:21, 7.49s/it]
Caching latents to disk: 39%|###9 | 48/122 [05:15<08:55, 7.24s/it]
Caching latents to disk: 39%|###9 | 48/122 [05:15<08:55, 7.24s/it]
Caching latents to disk: 40%|#### | 49/122 [05:22<08:47, 7.22s/it]
Caching latents to disk: 40%|#### | 49/122 [05:22<08:47, 7.22s/it]
Caching latents to disk: 41%|#### | 50/122 [05:29<08:34, 7.14s/it]
Caching latents to disk: 41%|#### | 50/122 [05:29<08:34, 7.14s/it]
Caching latents to disk: 42%|####1 | 51/122 [05:36<08:36, 7.28s/it]
Caching latents to disk: 42%|####1 | 51/122 [05:36<08:36, 7.28s/it]
Caching latents to disk: 43%|####2 | 52/122 [05:43<08:17, 7.10s/it]
Caching latents to disk: 43%|####2 | 52/122 [05:43<08:17, 7.10s/it]
Caching latents to disk: 43%|####3 | 53/122 [05:50<08:10, 7.11s/it]
Caching latents to disk: 43%|####3 | 53/122 [05:50<08:10, 7.11s/it]
Caching latents to disk: 44%|####4 | 54/122 [05:58<08:17, 7.31s/it]
Caching latents to disk: 44%|####4 | 54/122 [05:58<08:17, 7.31s/it]
Caching latents to disk: 45%|####5 | 55/122 [06:05<08:02, 7.20s/it]
Caching latents to disk: 45%|####5 | 55/122 [06:05<08:02, 7.20s/it]
Caching latents to disk: 46%|####5 | 56/122 [06:12<07:45, 7.06s/it]
Caching latents to disk: 46%|####5 | 56/122 [06:12<07:45, 7.06s/it]
Caching latents to disk: 47%|####6 | 57/122 [06:18<07:33, 6.98s/it]
Caching latents to disk: 47%|####6 | 57/122 [06:18<07:33, 6.98s/it]
Caching latents to disk: 48%|####7 | 58/122 [06:26<07:39, 7.18s/it]
Caching latents to disk: 48%|####7 | 58/122 [06:26<07:39, 7.18s/it]
Caching latents to disk: 48%|####8 | 59/122 [06:34<07:49, 7.46s/it]
Caching latents to disk: 48%|####8 | 59/122 [06:34<07:49, 7.46s/it]
Caching latents to disk: 49%|####9 | 60/122 [06:41<07:29, 7.25s/it]
Caching latents to disk: 49%|####9 | 60/122 [06:41<07:29, 7.25s/it]
Caching latents to disk: 50%|##### | 61/122 [06:48<07:25, 7.31s/it]
Caching latents to disk: 50%|##### | 61/122 [06:48<07:25, 7.31s/it]
Caching latents to disk: 100%|##########| 122/122 [06:48<00:00, 3.35s/it]
Caching latents to disk: 100%|##########| 122/122 [06:48<00:00, 3.35s/it] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/73 |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/122 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/122 [00:00<?, ?it/s]
Caching text embeddings to disk: 1%| | 1/122 [00:05<11:26, 5.68s/it]
Caching text embeddings to disk: 1%| | 1/122 [00:05<11:26, 5.68s/it]
Caching text embeddings to disk: 2%|1 | 2/122 [00:06<05:28, 2.74s/it]
Caching text embeddings to disk: 2%|1 | 2/122 [00:06<05:28, 2.74s/it]
Caching text embeddings to disk: 2%|2 | 3/122 [00:07<03:33, 1.79s/it]
Caching text embeddings to disk: 2%|2 | 3/122 [00:07<03:33, 1.79s/it]
Caching text embeddings to disk: 3%|3 | 4/122 [00:07<02:38, 1.34s/it]
Caching text embeddings to disk: 3%|3 | 4/122 [00:07<02:38, 1.34s/it]
Caching text embeddings to disk: 4%|4 | 5/122 [00:08<02:08, 1.10s/it]
Caching text embeddings to disk: 4%|4 | 5/122 [00:08<02:08, 1.10s/it]
Caching text embeddings to disk: 5%|4 | 6/122 [00:09<01:50, 1.05it/s]
Caching text embeddings to disk: 5%|4 | 6/122 [00:09<01:50, 1.05it/s]
Caching text embeddings to disk: 6%|5 | 7/122 [00:09<01:39, 1.16it/s]
Caching text embeddings to disk: 6%|5 | 7/122 [00:09<01:39, 1.16it/s]
Caching text embeddings to disk: 7%|6 | 8/122 [00:10<01:30, 1.26it/s]
Caching text embeddings to disk: 7%|6 | 8/122 [00:10<01:30, 1.26it/s]
Caching text embeddings to disk: 7%|7 | 9/122 [00:10<01:24, 1.33it/s]
Caching text embeddings to disk: 7%|7 | 9/122 [00:10<01:24, 1.33it/s]
Caching text embeddings to disk: 8%|8 | 10/122 [00:11<01:22, 1.36it/s]
Caching text embeddings to disk: 8%|8 | 10/122 [00:11<01:22, 1.36it/s]
Caching text embeddings to disk: 9%|9 | 11/122 [00:12<01:18, 1.41it/s]
Caching text embeddings to disk: 9%|9 | 11/122 [00:12<01:18, 1.41it/s]
Caching text embeddings to disk: 10%|9 | 12/122 [00:13<01:16, 1.44it/s]
Caching text embeddings to disk: 10%|9 | 12/122 [00:13<01:16, 1.44it/s]
Caching text embeddings to disk: 11%|# | 13/122 [00:13<01:14, 1.46it/s]
Caching text embeddings to disk: 11%|# | 13/122 [00:13<01:14, 1.46it/s]
Caching text embeddings to disk: 11%|#1 | 14/122 [00:14<01:13, 1.48it/s]
Caching text embeddings to disk: 11%|#1 | 14/122 [00:14<01:13, 1.48it/s]
Caching text embeddings to disk: 12%|#2 | 15/122 [00:14<01:11, 1.49it/s]
Caching text embeddings to disk: 12%|#2 | 15/122 [00:14<01:11, 1.49it/s]
Caching text embeddings to disk: 13%|#3 | 16/122 [00:15<01:11, 1.49it/s]
Caching text embeddings to disk: 13%|#3 | 16/122 [00:15<01:11, 1.49it/s]
Caching text embeddings to disk: 14%|#3 | 17/122 [00:16<01:11, 1.48it/s]
Caching text embeddings to disk: 14%|#3 | 17/122 [00:16<01:11, 1.48it/s]
Caching text embeddings to disk: 15%|#4 | 18/122 [00:17<01:10, 1.49it/s]
Caching text embeddings to disk: 15%|#4 | 18/122 [00:17<01:10, 1.49it/s]
Caching text embeddings to disk: 16%|#5 | 19/122 [00:17<01:09, 1.49it/s]
Caching text embeddings to disk: 16%|#5 | 19/122 [00:17<01:09, 1.49it/s]
Caching text embeddings to disk: 16%|#6 | 20/122 [00:18<01:08, 1.50it/s]
Caching text embeddings to disk: 16%|#6 | 20/122 [00:18<01:08, 1.50it/s]
Caching text embeddings to disk: 17%|#7 | 21/122 [00:18<01:07, 1.50it/s]
Caching text embeddings to disk: 17%|#7 | 21/122 [00:18<01:07, 1.50it/s]
Caching text embeddings to disk: 18%|#8 | 22/122 [00:19<01:07, 1.48it/s]
Caching text embeddings to disk: 18%|#8 | 22/122 [00:19<01:07, 1.48it/s]
Caching text embeddings to disk: 19%|#8 | 23/122 [00:20<01:06, 1.48it/s]
Caching text embeddings to disk: 19%|#8 | 23/122 [00:20<01:06, 1.48it/s]
Caching text embeddings to disk: 20%|#9 | 24/122 [00:21<01:05, 1.49it/s]
Caching text embeddings to disk: 20%|#9 | 24/122 [00:21<01:05, 1.49it/s]
Caching text embeddings to disk: 20%|## | 25/122 [00:21<01:04, 1.50it/s]
Caching text embeddings to disk: 20%|## | 25/122 [00:21<01:04, 1.50it/s]
Caching text embeddings to disk: 21%|##1 | 26/122 [00:22<01:03, 1.50it/s]
Caching text embeddings to disk: 21%|##1 | 26/122 [00:22<01:03, 1.50it/s]
Caching text embeddings to disk: 22%|##2 | 27/122 [00:22<01:02, 1.51it/s]
Caching text embeddings to disk: 22%|##2 | 27/122 [00:22<01:02, 1.51it/s]
Caching text embeddings to disk: 23%|##2 | 28/122 [00:23<01:02, 1.51it/s]
Caching text embeddings to disk: 23%|##2 | 28/122 [00:23<01:02, 1.51it/s]
Caching text embeddings to disk: 24%|##3 | 29/122 [00:24<00:59, 1.55it/s]
Caching text embeddings to disk: 24%|##3 | 29/122 [00:24<00:59, 1.55it/s]
Caching text embeddings to disk: 25%|##4 | 30/122 [00:24<01:00, 1.52it/s]
Caching text embeddings to disk: 25%|##4 | 30/122 [00:24<01:00, 1.52it/s]
Caching text embeddings to disk: 25%|##5 | 31/122 [00:25<01:00, 1.50it/s]
Caching text embeddings to disk: 25%|##5 | 31/122 [00:25<01:00, 1.50it/s]
Caching text embeddings to disk: 26%|##6 | 32/122 [00:26<00:59, 1.51it/s]
Caching text embeddings to disk: 26%|##6 | 32/122 [00:26<00:59, 1.51it/s]
Caching text embeddings to disk: 27%|##7 | 33/122 [00:26<00:59, 1.50it/s]
Caching text embeddings to disk: 27%|##7 | 33/122 [00:26<00:59, 1.50it/s]
Caching text embeddings to disk: 28%|##7 | 34/122 [00:27<00:59, 1.49it/s]
Caching text embeddings to disk: 28%|##7 | 34/122 [00:27<00:59, 1.49it/s]
Caching text embeddings to disk: 29%|##8 | 35/122 [00:28<00:57, 1.50it/s]
Caching text embeddings to disk: 29%|##8 | 35/122 [00:28<00:57, 1.50it/s]
Caching text embeddings to disk: 30%|##9 | 36/122 [00:29<00:58, 1.47it/s]
Caching text embeddings to disk: 30%|##9 | 36/122 [00:29<00:58, 1.47it/s]
Caching text embeddings to disk: 30%|### | 37/122 [00:29<00:57, 1.48it/s]
Caching text embeddings to disk: 30%|### | 37/122 [00:29<00:57, 1.48it/s]
Caching text embeddings to disk: 31%|###1 | 38/122 [00:30<00:56, 1.49it/s]
Caching text embeddings to disk: 31%|###1 | 38/122 [00:30<00:56, 1.49it/s]
Caching text embeddings to disk: 32%|###1 | 39/122 [00:31<00:56, 1.48it/s]
Caching text embeddings to disk: 32%|###1 | 39/122 [00:31<00:56, 1.48it/s]
Caching text embeddings to disk: 33%|###2 | 40/122 [00:31<00:55, 1.49it/s]
Caching text embeddings to disk: 33%|###2 | 40/122 [00:31<00:55, 1.49it/s]
Caching text embeddings to disk: 34%|###3 | 41/122 [00:32<00:54, 1.48it/s]
Caching text embeddings to disk: 34%|###3 | 41/122 [00:32<00:54, 1.48it/s]
Caching text embeddings to disk: 34%|###4 | 42/122 [00:33<00:53, 1.49it/s]
Caching text embeddings to disk: 34%|###4 | 42/122 [00:33<00:53, 1.49it/s]
Caching text embeddings to disk: 35%|###5 | 43/122 [00:33<00:53, 1.48it/s]
Caching text embeddings to disk: 35%|###5 | 43/122 [00:33<00:53, 1.48it/s]
Caching text embeddings to disk: 36%|###6 | 44/122 [00:34<00:52, 1.49it/s]
Caching text embeddings to disk: 36%|###6 | 44/122 [00:34<00:52, 1.49it/s]
Caching text embeddings to disk: 37%|###6 | 45/122 [00:35<00:51, 1.49it/s]
Caching text embeddings to disk: 37%|###6 | 45/122 [00:35<00:51, 1.49it/s]
Caching text embeddings to disk: 38%|###7 | 46/122 [00:35<00:50, 1.49it/s]
Caching text embeddings to disk: 38%|###7 | 46/122 [00:35<00:50, 1.49it/s]
Caching text embeddings to disk: 39%|###8 | 47/122 [00:36<00:49, 1.51it/s]
Caching text embeddings to disk: 39%|###8 | 47/122 [00:36<00:49, 1.51it/s]
Caching text embeddings to disk: 39%|###9 | 48/122 [00:37<00:49, 1.50it/s]
Caching text embeddings to disk: 39%|###9 | 48/122 [00:37<00:49, 1.50it/s]
Caching text embeddings to disk: 40%|#### | 49/122 [00:37<00:48, 1.49it/s]
Caching text embeddings to disk: 40%|#### | 49/122 [00:37<00:48, 1.49it/s]
Caching text embeddings to disk: 41%|#### | 50/122 [00:38<00:48, 1.49it/s]
Caching text embeddings to disk: 41%|#### | 50/122 [00:38<00:48, 1.49it/s]
Caching text embeddings to disk: 42%|####1 | 51/122 [00:39<00:47, 1.49it/s]
Caching text embeddings to disk: 42%|####1 | 51/122 [00:39<00:47, 1.49it/s]
Caching text embeddings to disk: 43%|####2 | 52/122 [00:39<00:46, 1.49it/s]
Caching text embeddings to disk: 43%|####2 | 52/122 [00:39<00:46, 1.49it/s]
Caching text embeddings to disk: 43%|####3 | 53/122 [00:40<00:46, 1.48it/s]
Caching text embeddings to disk: 43%|####3 | 53/122 [00:40<00:46, 1.48it/s]
Caching text embeddings to disk: 44%|####4 | 54/122 [00:41<00:45, 1.48it/s]
Caching text embeddings to disk: 44%|####4 | 54/122 [00:41<00:45, 1.48it/s]
Caching text embeddings to disk: 45%|####5 | 55/122 [00:41<00:45, 1.48it/s]
Caching text embeddings to disk: 45%|####5 | 55/122 [00:41<00:45, 1.48it/s]
Caching text embeddings to disk: 46%|####5 | 56/122 [00:42<00:44, 1.48it/s]
Caching text embeddings to disk: 46%|####5 | 56/122 [00:42<00:44, 1.48it/s]
Caching text embeddings to disk: 47%|####6 | 57/122 [00:43<00:43, 1.49it/s]
Caching text embeddings to disk: 47%|####6 | 57/122 [00:43<00:43, 1.49it/s]
Caching text embeddings to disk: 48%|####7 | 58/122 [00:43<00:42, 1.50it/s]
Caching text embeddings to disk: 48%|####7 | 58/122 [00:43<00:42, 1.50it/s]
Caching text embeddings to disk: 48%|####8 | 59/122 [00:44<00:42, 1.47it/s]
Caching text embeddings to disk: 48%|####8 | 59/122 [00:44<00:42, 1.47it/s]
Caching text embeddings to disk: 49%|####9 | 60/122 [00:45<00:41, 1.49it/s]
Caching text embeddings to disk: 49%|####9 | 60/122 [00:45<00:41, 1.49it/s]
Caching text embeddings to disk: 50%|##### | 61/122 [00:45<00:40, 1.51it/s]
Caching text embeddings to disk: 50%|##### | 61/122 [00:45<00:40, 1.51it/s]
Caching text embeddings to disk: 100%|##########| 122/122 [00:45<00:00, 2.67it/s]
Caching text embeddings to disk: 100%|##########| 122/122 [00:45<00:00, 2.67it/s] |
|
|
| Dataset: /app/ai-toolkit/datasets/81 |
| - Preprocessing video dimensions |
|
0%| | 0/2 [00:00<?, ?it/s]
0%| | 0/2 [00:00<?, ?it/s]
100%|##########| 2/2 [00:00<00:00, 219.25it/s]
100%|##########| 2/2 [00:00<00:00, 219.25it/s] |
|
|
| - Found 2 videos |
| Bucket sizes for /app/ai-toolkit/datasets/81: |
| 576x416: 2 files |
| 1 buckets made |
| Caching latents for /app/ai-toolkit/datasets/81 |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/2 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/2 [00:00<?, ?it/s]
Caching latents to disk: 50%|##### | 1/2 [00:06<00:06, 6.42s/it]
Caching latents to disk: 50%|##### | 1/2 [00:06<00:06, 6.42s/it]
Caching latents to disk: 100%|##########| 2/2 [00:06<00:00, 3.21s/it]
Caching latents to disk: 100%|##########| 2/2 [00:06<00:00, 3.21s/it] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/81 |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/2 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/2 [00:00<?, ?it/s]
Caching text embeddings to disk: 50%|##### | 1/2 [00:05<00:05, 5.17s/it]
Caching text embeddings to disk: 50%|##### | 1/2 [00:05<00:05, 5.17s/it]
Caching text embeddings to disk: 100%|##########| 2/2 [00:05<00:00, 2.59s/it]
Caching text embeddings to disk: 100%|##########| 2/2 [00:05<00:00, 2.59s/it] |
|
|
| Dataset: /app/ai-toolkit/datasets/89 |
| - Preprocessing video dimensions |
|
0%| | 0/4 [00:00<?, ?it/s]
0%| | 0/4 [00:00<?, ?it/s]
100%|##########| 4/4 [00:00<00:00, 266.08it/s]
100%|##########| 4/4 [00:00<00:00, 266.08it/s] |
|
|
| - Found 4 videos |
| Bucket sizes for /app/ai-toolkit/datasets/89: |
| 576x416: 4 files |
| 1 buckets made |
| Caching latents for /app/ai-toolkit/datasets/89 |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/4 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/4 [00:00<?, ?it/s]
Caching latents to disk: 25%|##5 | 1/4 [00:07<00:22, 7.63s/it]
Caching latents to disk: 25%|##5 | 1/4 [00:07<00:22, 7.63s/it]
Caching latents to disk: 50%|##### | 2/4 [00:15<00:15, 7.72s/it]
Caching latents to disk: 50%|##### | 2/4 [00:15<00:15, 7.72s/it]
Caching latents to disk: 100%|##########| 4/4 [00:15<00:00, 3.85s/it]
Caching latents to disk: 100%|##########| 4/4 [00:15<00:00, 3.85s/it] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/89 |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/4 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/4 [00:00<?, ?it/s]
Caching text embeddings to disk: 25%|##5 | 1/4 [00:04<00:14, 4.81s/it]
Caching text embeddings to disk: 25%|##5 | 1/4 [00:04<00:14, 4.81s/it]
Caching text embeddings to disk: 50%|##### | 2/4 [00:05<00:04, 2.37s/it]
Caching text embeddings to disk: 50%|##### | 2/4 [00:05<00:04, 2.37s/it]
Caching text embeddings to disk: 100%|##########| 4/4 [00:05<00:00, 1.37s/it]
Caching text embeddings to disk: 100%|##########| 4/4 [00:05<00:00, 1.37s/it] |
|
|
| Dataset: /app/ai-toolkit/datasets/97 |
| - Preprocessing video dimensions |
|
0%| | 0/58 [00:00<?, ?it/s]
0%| | 0/58 [00:00<?, ?it/s]
26%|##5 | 15/58 [00:00<00:00, 146.08it/s]
26%|##5 | 15/58 [00:00<00:00, 146.08it/s]
100%|##########| 58/58 [00:00<00:00, 288.51it/s]
100%|##########| 58/58 [00:00<00:00, 288.51it/s] |
|
|
| - Found 58 videos |
| Bucket sizes for /app/ai-toolkit/datasets/97: |
| 576x416: 46 files |
| 672x384: 4 files |
| 576x448: 8 files |
| 3 buckets made |
| Caching latents for /app/ai-toolkit/datasets/97 |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/58 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/58 [00:00<?, ?it/s]
Caching latents to disk: 2%|1 | 1/58 [00:07<07:09, 7.54s/it]
Caching latents to disk: 2%|1 | 1/58 [00:07<07:09, 7.54s/it]
Caching latents to disk: 3%|3 | 2/58 [00:15<07:16, 7.80s/it]
Caching latents to disk: 3%|3 | 2/58 [00:15<07:16, 7.80s/it]
Caching latents to disk: 5%|5 | 3/58 [00:23<07:16, 7.94s/it]
Caching latents to disk: 5%|5 | 3/58 [00:23<07:16, 7.94s/it]
Caching latents to disk: 7%|6 | 4/58 [00:33<07:56, 8.83s/it]
Caching latents to disk: 7%|6 | 4/58 [00:33<07:56, 8.83s/it]
Caching latents to disk: 9%|8 | 5/58 [00:43<08:10, 9.25s/it]
Caching latents to disk: 9%|8 | 5/58 [00:43<08:10, 9.25s/it]
Caching latents to disk: 10%|# | 6/58 [00:53<08:14, 9.51s/it]
Caching latents to disk: 10%|# | 6/58 [00:53<08:14, 9.51s/it]
Caching latents to disk: 12%|#2 | 7/58 [01:03<08:13, 9.67s/it]
Caching latents to disk: 12%|#2 | 7/58 [01:03<08:13, 9.67s/it]
Caching latents to disk: 14%|#3 | 8/58 [01:11<07:27, 8.95s/it]
Caching latents to disk: 14%|#3 | 8/58 [01:11<07:27, 8.95s/it]
Caching latents to disk: 16%|#5 | 9/58 [01:21<07:36, 9.31s/it]
Caching latents to disk: 16%|#5 | 9/58 [01:21<07:36, 9.31s/it]
Caching latents to disk: 17%|#7 | 10/58 [01:30<07:22, 9.22s/it]
Caching latents to disk: 17%|#7 | 10/58 [01:30<07:22, 9.22s/it]
Caching latents to disk: 19%|#8 | 11/58 [01:37<06:44, 8.60s/it]
Caching latents to disk: 19%|#8 | 11/58 [01:37<06:44, 8.60s/it]
Caching latents to disk: 21%|## | 12/58 [01:47<06:56, 9.05s/it]
Caching latents to disk: 21%|## | 12/58 [01:47<06:56, 9.05s/it]
Caching latents to disk: 22%|##2 | 13/58 [02:01<07:54, 10.53s/it]
Caching latents to disk: 22%|##2 | 13/58 [02:01<07:54, 10.53s/it]
Caching latents to disk: 24%|##4 | 14/58 [02:09<07:03, 9.63s/it]
Caching latents to disk: 24%|##4 | 14/58 [02:09<07:03, 9.63s/it]
Caching latents to disk: 26%|##5 | 15/58 [02:19<07:00, 9.78s/it]
Caching latents to disk: 26%|##5 | 15/58 [02:19<07:00, 9.78s/it]
Caching latents to disk: 28%|##7 | 16/58 [02:28<06:42, 9.60s/it]
Caching latents to disk: 28%|##7 | 16/58 [02:28<06:42, 9.60s/it]
Caching latents to disk: 29%|##9 | 17/58 [02:39<06:47, 9.94s/it]
Caching latents to disk: 29%|##9 | 17/58 [02:39<06:47, 9.94s/it]
Caching latents to disk: 31%|###1 | 18/58 [02:47<06:13, 9.34s/it]
Caching latents to disk: 31%|###1 | 18/58 [02:47<06:13, 9.34s/it]
Caching latents to disk: 33%|###2 | 19/58 [02:57<06:14, 9.60s/it]
Caching latents to disk: 33%|###2 | 19/58 [02:57<06:14, 9.60s/it]
Caching latents to disk: 34%|###4 | 20/58 [03:06<06:03, 9.56s/it]
Caching latents to disk: 34%|###4 | 20/58 [03:06<06:03, 9.56s/it]
Caching latents to disk: 36%|###6 | 21/58 [03:17<06:06, 9.89s/it]
Caching latents to disk: 36%|###6 | 21/58 [03:17<06:06, 9.89s/it]
Caching latents to disk: 38%|###7 | 22/58 [03:27<06:01, 10.04s/it]
Caching latents to disk: 38%|###7 | 22/58 [03:27<06:01, 10.04s/it]
Caching latents to disk: 40%|###9 | 23/58 [03:38<05:58, 10.23s/it]
Caching latents to disk: 40%|###9 | 23/58 [03:38<05:58, 10.23s/it]
Caching latents to disk: 41%|####1 | 24/58 [03:46<05:29, 9.69s/it]
Caching latents to disk: 41%|####1 | 24/58 [03:46<05:29, 9.69s/it]
Caching latents to disk: 43%|####3 | 25/58 [03:57<05:24, 9.84s/it]
Caching latents to disk: 43%|####3 | 25/58 [03:57<05:24, 9.84s/it]
Caching latents to disk: 45%|####4 | 26/58 [04:06<05:10, 9.70s/it]
Caching latents to disk: 45%|####4 | 26/58 [04:06<05:10, 9.70s/it]
Caching latents to disk: 47%|####6 | 27/58 [04:16<05:02, 9.75s/it]
Caching latents to disk: 47%|####6 | 27/58 [04:16<05:02, 9.75s/it]
Caching latents to disk: 48%|####8 | 28/58 [04:26<04:59, 9.99s/it]
Caching latents to disk: 48%|####8 | 28/58 [04:26<04:59, 9.99s/it]
Caching latents to disk: 50%|##### | 29/58 [04:37<04:51, 10.05s/it]
Caching latents to disk: 50%|##### | 29/58 [04:37<04:51, 10.05s/it]
Caching latents to disk: 100%|##########| 58/58 [04:37<00:00, 4.78s/it]
Caching latents to disk: 100%|##########| 58/58 [04:37<00:00, 4.78s/it] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/97 |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/58 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/58 [00:00<?, ?it/s]
Caching text embeddings to disk: 2%|1 | 1/58 [00:04<04:39, 4.90s/it]
Caching text embeddings to disk: 2%|1 | 1/58 [00:04<04:39, 4.90s/it]
Caching text embeddings to disk: 3%|3 | 2/58 [00:05<02:15, 2.42s/it]
Caching text embeddings to disk: 3%|3 | 2/58 [00:05<02:15, 2.42s/it]
Caching text embeddings to disk: 5%|5 | 3/58 [00:06<01:29, 1.62s/it]
Caching text embeddings to disk: 5%|5 | 3/58 [00:06<01:29, 1.62s/it]
Caching text embeddings to disk: 7%|6 | 4/58 [00:06<01:07, 1.25s/it]
Caching text embeddings to disk: 7%|6 | 4/58 [00:06<01:07, 1.25s/it]
Caching text embeddings to disk: 9%|8 | 5/58 [00:07<00:54, 1.04s/it]
Caching text embeddings to disk: 9%|8 | 5/58 [00:07<00:54, 1.04s/it]
Caching text embeddings to disk: 10%|# | 6/58 [00:08<00:46, 1.12it/s]
Caching text embeddings to disk: 10%|# | 6/58 [00:08<00:46, 1.12it/s]
Caching text embeddings to disk: 12%|#2 | 7/58 [00:08<00:41, 1.23it/s]
Caching text embeddings to disk: 12%|#2 | 7/58 [00:08<00:41, 1.23it/s]
Caching text embeddings to disk: 14%|#3 | 8/58 [00:09<00:38, 1.31it/s]
Caching text embeddings to disk: 14%|#3 | 8/58 [00:09<00:38, 1.31it/s]
Caching text embeddings to disk: 16%|#5 | 9/58 [00:10<00:35, 1.37it/s]
Caching text embeddings to disk: 16%|#5 | 9/58 [00:10<00:35, 1.37it/s]
Caching text embeddings to disk: 17%|#7 | 10/58 [00:10<00:34, 1.40it/s]
Caching text embeddings to disk: 17%|#7 | 10/58 [00:10<00:34, 1.40it/s]
Caching text embeddings to disk: 19%|#8 | 11/58 [00:11<00:32, 1.44it/s]
Caching text embeddings to disk: 19%|#8 | 11/58 [00:11<00:32, 1.44it/s]
Caching text embeddings to disk: 21%|## | 12/58 [00:12<00:31, 1.45it/s]
Caching text embeddings to disk: 21%|## | 12/58 [00:12<00:31, 1.45it/s]
Caching text embeddings to disk: 22%|##2 | 13/58 [00:12<00:30, 1.46it/s]
Caching text embeddings to disk: 22%|##2 | 13/58 [00:12<00:30, 1.46it/s]
Caching text embeddings to disk: 24%|##4 | 14/58 [00:13<00:30, 1.46it/s]
Caching text embeddings to disk: 24%|##4 | 14/58 [00:13<00:30, 1.46it/s]
Caching text embeddings to disk: 26%|##5 | 15/58 [00:14<00:29, 1.47it/s]
Caching text embeddings to disk: 26%|##5 | 15/58 [00:14<00:29, 1.47it/s]
Caching text embeddings to disk: 28%|##7 | 16/58 [00:14<00:28, 1.49it/s]
Caching text embeddings to disk: 28%|##7 | 16/58 [00:14<00:28, 1.49it/s]
Caching text embeddings to disk: 29%|##9 | 17/58 [00:15<00:27, 1.49it/s]
Caching text embeddings to disk: 29%|##9 | 17/58 [00:15<00:27, 1.49it/s]
Caching text embeddings to disk: 31%|###1 | 18/58 [00:16<00:26, 1.51it/s]
Caching text embeddings to disk: 31%|###1 | 18/58 [00:16<00:26, 1.51it/s]
Caching text embeddings to disk: 33%|###2 | 19/58 [00:16<00:25, 1.52it/s]
Caching text embeddings to disk: 33%|###2 | 19/58 [00:16<00:25, 1.52it/s]
Caching text embeddings to disk: 34%|###4 | 20/58 [00:17<00:24, 1.53it/s]
Caching text embeddings to disk: 34%|###4 | 20/58 [00:17<00:24, 1.53it/s]
Caching text embeddings to disk: 36%|###6 | 21/58 [00:18<00:24, 1.52it/s]
Caching text embeddings to disk: 36%|###6 | 21/58 [00:18<00:24, 1.52it/s]
Caching text embeddings to disk: 38%|###7 | 22/58 [00:18<00:23, 1.50it/s]
Caching text embeddings to disk: 38%|###7 | 22/58 [00:18<00:23, 1.50it/s]
Caching text embeddings to disk: 40%|###9 | 23/58 [00:19<00:23, 1.50it/s]
Caching text embeddings to disk: 40%|###9 | 23/58 [00:19<00:23, 1.50it/s]
Caching text embeddings to disk: 41%|####1 | 24/58 [00:20<00:22, 1.52it/s]
Caching text embeddings to disk: 41%|####1 | 24/58 [00:20<00:22, 1.52it/s]
Caching text embeddings to disk: 43%|####3 | 25/58 [00:20<00:21, 1.51it/s]
Caching text embeddings to disk: 43%|####3 | 25/58 [00:20<00:21, 1.51it/s]
Caching text embeddings to disk: 45%|####4 | 26/58 [00:21<00:21, 1.51it/s]
Caching text embeddings to disk: 45%|####4 | 26/58 [00:21<00:21, 1.51it/s]
Caching text embeddings to disk: 47%|####6 | 27/58 [00:22<00:20, 1.52it/s]
Caching text embeddings to disk: 47%|####6 | 27/58 [00:22<00:20, 1.52it/s]
Caching text embeddings to disk: 48%|####8 | 28/58 [00:22<00:19, 1.53it/s]
Caching text embeddings to disk: 48%|####8 | 28/58 [00:22<00:19, 1.53it/s]
Caching text embeddings to disk: 50%|##### | 29/58 [00:23<00:19, 1.50it/s]
Caching text embeddings to disk: 50%|##### | 29/58 [00:23<00:19, 1.50it/s]
Caching text embeddings to disk: 100%|##########| 58/58 [00:23<00:00, 2.47it/s]
Caching text embeddings to disk: 100%|##########| 58/58 [00:23<00:00, 2.47it/s] |
|
|
| Dataset: /app/ai-toolkit/datasets/images |
| - Preprocessing image dimensions |
|
0%| | 0/47 [00:00<?, ?it/s]
0%| | 0/47 [00:00<?, ?it/s]
70%|####### | 33/47 [00:00<00:00, 329.71it/s]
70%|####### | 33/47 [00:00<00:00, 329.71it/s]
100%|##########| 47/47 [00:00<00:00, 300.62it/s]
100%|##########| 47/47 [00:00<00:00, 300.62it/s] |
|
|
| - Found 47 images |
| Bucket sizes for /app/ai-toolkit/datasets/images: |
| 512x512: 46 files |
| 576x416: 1 files |
| 2 buckets made |
| Caching latents for /app/ai-toolkit/datasets/images |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching latents to disk: 2%|2 | 1/47 [00:00<00:05, 7.72it/s]
Caching latents to disk: 2%|2 | 1/47 [00:00<00:05, 7.72it/s]
Caching latents to disk: 4%|4 | 2/47 [00:00<00:05, 7.66it/s]
Caching latents to disk: 4%|4 | 2/47 [00:00<00:05, 7.66it/s]
Caching latents to disk: 6%|6 | 3/47 [00:00<00:05, 7.72it/s]
Caching latents to disk: 6%|6 | 3/47 [00:00<00:05, 7.72it/s]
Caching latents to disk: 9%|8 | 4/47 [00:00<00:05, 7.78it/s]
Caching latents to disk: 9%|8 | 4/47 [00:00<00:05, 7.78it/s]
Caching latents to disk: 11%|# | 5/47 [00:00<00:05, 7.78it/s]
Caching latents to disk: 11%|# | 5/47 [00:00<00:05, 7.78it/s]
Caching latents to disk: 13%|#2 | 6/47 [00:00<00:05, 7.74it/s]
Caching latents to disk: 13%|#2 | 6/47 [00:00<00:05, 7.74it/s]
Caching latents to disk: 15%|#4 | 7/47 [00:00<00:05, 7.74it/s]
Caching latents to disk: 15%|#4 | 7/47 [00:00<00:05, 7.74it/s]
Caching latents to disk: 17%|#7 | 8/47 [00:01<00:05, 7.72it/s]
Caching latents to disk: 17%|#7 | 8/47 [00:01<00:05, 7.72it/s]
Caching latents to disk: 19%|#9 | 9/47 [00:01<00:04, 7.77it/s]
Caching latents to disk: 19%|#9 | 9/47 [00:01<00:04, 7.77it/s]
Caching latents to disk: 21%|##1 | 10/47 [00:01<00:04, 7.47it/s]
Caching latents to disk: 21%|##1 | 10/47 [00:01<00:04, 7.47it/s]
Caching latents to disk: 23%|##3 | 11/47 [00:01<00:04, 7.52it/s]
Caching latents to disk: 23%|##3 | 11/47 [00:01<00:04, 7.52it/s]
Caching latents to disk: 26%|##5 | 12/47 [00:01<00:04, 7.29it/s]
Caching latents to disk: 26%|##5 | 12/47 [00:01<00:04, 7.29it/s]
Caching latents to disk: 28%|##7 | 13/47 [00:01<00:04, 7.39it/s]
Caching latents to disk: 28%|##7 | 13/47 [00:01<00:04, 7.39it/s]
Caching latents to disk: 30%|##9 | 14/47 [00:01<00:04, 7.49it/s]
Caching latents to disk: 30%|##9 | 14/47 [00:01<00:04, 7.49it/s]
Caching latents to disk: 32%|###1 | 15/47 [00:01<00:04, 7.58it/s]
Caching latents to disk: 32%|###1 | 15/47 [00:01<00:04, 7.58it/s]
Caching latents to disk: 34%|###4 | 16/47 [00:02<00:04, 7.63it/s]
Caching latents to disk: 34%|###4 | 16/47 [00:02<00:04, 7.63it/s]
Caching latents to disk: 36%|###6 | 17/47 [00:02<00:03, 7.69it/s]
Caching latents to disk: 36%|###6 | 17/47 [00:02<00:03, 7.69it/s]
Caching latents to disk: 38%|###8 | 18/47 [00:02<00:03, 7.70it/s]
Caching latents to disk: 38%|###8 | 18/47 [00:02<00:03, 7.70it/s]
Caching latents to disk: 40%|#### | 19/47 [00:02<00:03, 7.71it/s]
Caching latents to disk: 40%|#### | 19/47 [00:02<00:03, 7.71it/s]
Caching latents to disk: 43%|####2 | 20/47 [00:02<00:03, 7.50it/s]
Caching latents to disk: 43%|####2 | 20/47 [00:02<00:03, 7.50it/s]
Caching latents to disk: 45%|####4 | 21/47 [00:02<00:03, 7.58it/s]
Caching latents to disk: 45%|####4 | 21/47 [00:02<00:03, 7.58it/s]
Caching latents to disk: 47%|####6 | 22/47 [00:02<00:03, 7.65it/s]
Caching latents to disk: 47%|####6 | 22/47 [00:02<00:03, 7.65it/s]
Caching latents to disk: 49%|####8 | 23/47 [00:03<00:03, 7.67it/s]
Caching latents to disk: 49%|####8 | 23/47 [00:03<00:03, 7.67it/s]
Caching latents to disk: 51%|#####1 | 24/47 [00:03<00:02, 7.70it/s]
Caching latents to disk: 51%|#####1 | 24/47 [00:03<00:02, 7.70it/s]
Caching latents to disk: 53%|#####3 | 25/47 [00:03<00:02, 7.70it/s]
Caching latents to disk: 53%|#####3 | 25/47 [00:03<00:02, 7.70it/s]
Caching latents to disk: 55%|#####5 | 26/47 [00:03<00:02, 7.65it/s]
Caching latents to disk: 55%|#####5 | 26/47 [00:03<00:02, 7.65it/s]
Caching latents to disk: 57%|#####7 | 27/47 [00:03<00:02, 7.71it/s]
Caching latents to disk: 57%|#####7 | 27/47 [00:03<00:02, 7.71it/s]
Caching latents to disk: 60%|#####9 | 28/47 [00:03<00:02, 7.32it/s]
Caching latents to disk: 60%|#####9 | 28/47 [00:03<00:02, 7.32it/s]
Caching latents to disk: 62%|######1 | 29/47 [00:03<00:02, 7.44it/s]
Caching latents to disk: 62%|######1 | 29/47 [00:03<00:02, 7.44it/s]
Caching latents to disk: 64%|######3 | 30/47 [00:03<00:02, 7.53it/s]
Caching latents to disk: 64%|######3 | 30/47 [00:03<00:02, 7.53it/s]
Caching latents to disk: 66%|######5 | 31/47 [00:04<00:02, 7.61it/s]
Caching latents to disk: 66%|######5 | 31/47 [00:04<00:02, 7.61it/s]
Caching latents to disk: 68%|######8 | 32/47 [00:04<00:01, 7.67it/s]
Caching latents to disk: 68%|######8 | 32/47 [00:04<00:01, 7.67it/s]
Caching latents to disk: 70%|####### | 33/47 [00:04<00:01, 7.62it/s]
Caching latents to disk: 70%|####### | 33/47 [00:04<00:01, 7.62it/s]
Caching latents to disk: 72%|#######2 | 34/47 [00:04<00:01, 7.64it/s]
Caching latents to disk: 72%|#######2 | 34/47 [00:04<00:01, 7.64it/s]
Caching latents to disk: 74%|#######4 | 35/47 [00:04<00:01, 7.45it/s]
Caching latents to disk: 74%|#######4 | 35/47 [00:04<00:01, 7.45it/s]
Caching latents to disk: 77%|#######6 | 36/47 [00:04<00:01, 7.45it/s]
Caching latents to disk: 77%|#######6 | 36/47 [00:04<00:01, 7.45it/s]
Caching latents to disk: 79%|#######8 | 37/47 [00:04<00:01, 7.35it/s]
Caching latents to disk: 79%|#######8 | 37/47 [00:04<00:01, 7.35it/s]
Caching latents to disk: 81%|######## | 38/47 [00:05<00:01, 6.99it/s]
Caching latents to disk: 81%|######## | 38/47 [00:05<00:01, 6.99it/s]
Caching latents to disk: 83%|########2 | 39/47 [00:05<00:01, 7.27it/s]
Caching latents to disk: 83%|########2 | 39/47 [00:05<00:01, 7.27it/s]
Caching latents to disk: 85%|########5 | 40/47 [00:05<00:00, 7.43it/s]
Caching latents to disk: 85%|########5 | 40/47 [00:05<00:00, 7.43it/s]
Caching latents to disk: 87%|########7 | 41/47 [00:05<00:00, 7.54it/s]
Caching latents to disk: 87%|########7 | 41/47 [00:05<00:00, 7.54it/s]
Caching latents to disk: 89%|########9 | 42/47 [00:05<00:00, 7.57it/s]
Caching latents to disk: 89%|########9 | 42/47 [00:05<00:00, 7.57it/s]
Caching latents to disk: 91%|#########1| 43/47 [00:05<00:00, 7.62it/s]
Caching latents to disk: 91%|#########1| 43/47 [00:05<00:00, 7.62it/s]
Caching latents to disk: 94%|#########3| 44/47 [00:05<00:00, 7.65it/s]
Caching latents to disk: 94%|#########3| 44/47 [00:05<00:00, 7.65it/s]
Caching latents to disk: 96%|#########5| 45/47 [00:05<00:00, 7.68it/s]
Caching latents to disk: 96%|#########5| 45/47 [00:05<00:00, 7.68it/s]
Caching latents to disk: 98%|#########7| 46/47 [00:06<00:00, 7.49it/s]
Caching latents to disk: 98%|#########7| 46/47 [00:06<00:00, 7.49it/s]
Caching latents to disk: 100%|##########| 47/47 [00:06<00:00, 7.51it/s]
Caching latents to disk: 100%|##########| 47/47 [00:06<00:00, 7.51it/s]
Caching latents to disk: 100%|##########| 47/47 [00:06<00:00, 7.56it/s]
Caching latents to disk: 100%|##########| 47/47 [00:06<00:00, 7.56it/s] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/images |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching text embeddings to disk: 2%|2 | 1/47 [00:04<03:47, 4.94s/it]
Caching text embeddings to disk: 2%|2 | 1/47 [00:04<03:47, 4.94s/it]
Caching text embeddings to disk: 4%|4 | 2/47 [00:05<01:47, 2.39s/it]
Caching text embeddings to disk: 4%|4 | 2/47 [00:05<01:47, 2.39s/it]
Caching text embeddings to disk: 6%|6 | 3/47 [00:06<01:10, 1.61s/it]
Caching text embeddings to disk: 6%|6 | 3/47 [00:06<01:10, 1.61s/it]
Caching text embeddings to disk: 9%|8 | 4/47 [00:06<00:54, 1.26s/it]
Caching text embeddings to disk: 9%|8 | 4/47 [00:06<00:54, 1.26s/it]
Caching text embeddings to disk: 11%|# | 5/47 [00:07<00:44, 1.07s/it]
Caching text embeddings to disk: 11%|# | 5/47 [00:07<00:44, 1.07s/it]
Caching text embeddings to disk: 13%|#2 | 6/47 [00:08<00:39, 1.05it/s]
Caching text embeddings to disk: 13%|#2 | 6/47 [00:08<00:39, 1.05it/s]
Caching text embeddings to disk: 15%|#4 | 7/47 [00:09<00:34, 1.16it/s]
Caching text embeddings to disk: 15%|#4 | 7/47 [00:09<00:34, 1.16it/s]
Caching text embeddings to disk: 17%|#7 | 8/47 [00:09<00:31, 1.23it/s]
Caching text embeddings to disk: 17%|#7 | 8/47 [00:09<00:31, 1.23it/s]
Caching text embeddings to disk: 19%|#9 | 9/47 [00:10<00:29, 1.30it/s]
Caching text embeddings to disk: 19%|#9 | 9/47 [00:10<00:29, 1.30it/s]
Caching text embeddings to disk: 21%|##1 | 10/47 [00:11<00:27, 1.34it/s]
Caching text embeddings to disk: 21%|##1 | 10/47 [00:11<00:27, 1.34it/s]
Caching text embeddings to disk: 23%|##3 | 11/47 [00:11<00:26, 1.37it/s]
Caching text embeddings to disk: 23%|##3 | 11/47 [00:11<00:26, 1.37it/s]
Caching text embeddings to disk: 26%|##5 | 12/47 [00:12<00:25, 1.39it/s]
Caching text embeddings to disk: 26%|##5 | 12/47 [00:12<00:25, 1.39it/s]
Caching text embeddings to disk: 28%|##7 | 13/47 [00:13<00:23, 1.43it/s]
Caching text embeddings to disk: 28%|##7 | 13/47 [00:13<00:23, 1.43it/s]
Caching text embeddings to disk: 30%|##9 | 14/47 [00:13<00:22, 1.44it/s]
Caching text embeddings to disk: 30%|##9 | 14/47 [00:13<00:22, 1.44it/s]
Caching text embeddings to disk: 32%|###1 | 15/47 [00:14<00:22, 1.44it/s]
Caching text embeddings to disk: 32%|###1 | 15/47 [00:14<00:22, 1.44it/s]
Caching text embeddings to disk: 34%|###4 | 16/47 [00:15<00:21, 1.45it/s]
Caching text embeddings to disk: 34%|###4 | 16/47 [00:15<00:21, 1.45it/s]
Caching text embeddings to disk: 36%|###6 | 17/47 [00:15<00:20, 1.44it/s]
Caching text embeddings to disk: 36%|###6 | 17/47 [00:15<00:20, 1.44it/s]
Caching text embeddings to disk: 38%|###8 | 18/47 [00:16<00:19, 1.46it/s]
Caching text embeddings to disk: 38%|###8 | 18/47 [00:16<00:19, 1.46it/s]
Caching text embeddings to disk: 40%|#### | 19/47 [00:17<00:19, 1.45it/s]
Caching text embeddings to disk: 40%|#### | 19/47 [00:17<00:19, 1.45it/s]
Caching text embeddings to disk: 43%|####2 | 20/47 [00:18<00:18, 1.45it/s]
Caching text embeddings to disk: 43%|####2 | 20/47 [00:18<00:18, 1.45it/s]
Caching text embeddings to disk: 45%|####4 | 21/47 [00:18<00:18, 1.44it/s]
Caching text embeddings to disk: 45%|####4 | 21/47 [00:18<00:18, 1.44it/s]
Caching text embeddings to disk: 47%|####6 | 22/47 [00:19<00:17, 1.43it/s]
Caching text embeddings to disk: 47%|####6 | 22/47 [00:19<00:17, 1.43it/s]
Caching text embeddings to disk: 49%|####8 | 23/47 [00:20<00:16, 1.42it/s]
Caching text embeddings to disk: 49%|####8 | 23/47 [00:20<00:16, 1.42it/s]
Caching text embeddings to disk: 51%|#####1 | 24/47 [00:20<00:16, 1.42it/s]
Caching text embeddings to disk: 51%|#####1 | 24/47 [00:20<00:16, 1.42it/s]
Caching text embeddings to disk: 53%|#####3 | 25/47 [00:21<00:15, 1.43it/s]
Caching text embeddings to disk: 53%|#####3 | 25/47 [00:21<00:15, 1.43it/s]
Caching text embeddings to disk: 55%|#####5 | 26/47 [00:22<00:14, 1.43it/s]
Caching text embeddings to disk: 55%|#####5 | 26/47 [00:22<00:14, 1.43it/s]
Caching text embeddings to disk: 57%|#####7 | 27/47 [00:22<00:13, 1.43it/s]
Caching text embeddings to disk: 57%|#####7 | 27/47 [00:22<00:13, 1.43it/s]
Caching text embeddings to disk: 60%|#####9 | 28/47 [00:23<00:13, 1.44it/s]
Caching text embeddings to disk: 60%|#####9 | 28/47 [00:23<00:13, 1.44it/s]
Caching text embeddings to disk: 62%|######1 | 29/47 [00:24<00:12, 1.44it/s]
Caching text embeddings to disk: 62%|######1 | 29/47 [00:24<00:12, 1.44it/s]
Caching text embeddings to disk: 64%|######3 | 30/47 [00:25<00:11, 1.44it/s]
Caching text embeddings to disk: 64%|######3 | 30/47 [00:25<00:11, 1.44it/s]
Caching text embeddings to disk: 66%|######5 | 31/47 [00:25<00:11, 1.45it/s]
Caching text embeddings to disk: 66%|######5 | 31/47 [00:25<00:11, 1.45it/s]
Caching text embeddings to disk: 68%|######8 | 32/47 [00:26<00:10, 1.46it/s]
Caching text embeddings to disk: 68%|######8 | 32/47 [00:26<00:10, 1.46it/s]
Caching text embeddings to disk: 70%|####### | 33/47 [00:27<00:09, 1.48it/s]
Caching text embeddings to disk: 70%|####### | 33/47 [00:27<00:09, 1.48it/s]
Caching text embeddings to disk: 72%|#######2 | 34/47 [00:27<00:08, 1.50it/s]
Caching text embeddings to disk: 72%|#######2 | 34/47 [00:27<00:08, 1.50it/s]
Caching text embeddings to disk: 74%|#######4 | 35/47 [00:28<00:07, 1.51it/s]
Caching text embeddings to disk: 74%|#######4 | 35/47 [00:28<00:07, 1.51it/s]
Caching text embeddings to disk: 77%|#######6 | 36/47 [00:29<00:07, 1.49it/s]
Caching text embeddings to disk: 77%|#######6 | 36/47 [00:29<00:07, 1.49it/s]
Caching text embeddings to disk: 79%|#######8 | 37/47 [00:29<00:06, 1.49it/s]
Caching text embeddings to disk: 79%|#######8 | 37/47 [00:29<00:06, 1.49it/s]
Caching text embeddings to disk: 81%|######## | 38/47 [00:30<00:05, 1.51it/s]
Caching text embeddings to disk: 81%|######## | 38/47 [00:30<00:05, 1.51it/s]
Caching text embeddings to disk: 83%|########2 | 39/47 [00:30<00:05, 1.52it/s]
Caching text embeddings to disk: 83%|########2 | 39/47 [00:30<00:05, 1.52it/s]
Caching text embeddings to disk: 85%|########5 | 40/47 [00:31<00:04, 1.56it/s]
Caching text embeddings to disk: 85%|########5 | 40/47 [00:31<00:04, 1.56it/s]
Caching text embeddings to disk: 87%|########7 | 41/47 [00:32<00:03, 1.58it/s]
Caching text embeddings to disk: 87%|########7 | 41/47 [00:32<00:03, 1.58it/s]
Caching text embeddings to disk: 89%|########9 | 42/47 [00:32<00:03, 1.56it/s]
Caching text embeddings to disk: 89%|########9 | 42/47 [00:32<00:03, 1.56it/s]
Caching text embeddings to disk: 91%|#########1| 43/47 [00:33<00:02, 1.52it/s]
Caching text embeddings to disk: 91%|#########1| 43/47 [00:33<00:02, 1.52it/s]
Caching text embeddings to disk: 94%|#########3| 44/47 [00:34<00:02, 1.50it/s]
Caching text embeddings to disk: 94%|#########3| 44/47 [00:34<00:02, 1.50it/s]
Caching text embeddings to disk: 96%|#########5| 45/47 [00:34<00:01, 1.50it/s]
Caching text embeddings to disk: 96%|#########5| 45/47 [00:34<00:01, 1.50it/s]
Caching text embeddings to disk: 98%|#########7| 46/47 [00:35<00:00, 1.50it/s]
Caching text embeddings to disk: 98%|#########7| 46/47 [00:35<00:00, 1.50it/s]
Caching text embeddings to disk: 100%|##########| 47/47 [00:36<00:00, 1.47it/s]
Caching text embeddings to disk: 100%|##########| 47/47 [00:36<00:00, 1.47it/s]
Caching text embeddings to disk: 100%|##########| 47/47 [00:36<00:00, 1.30it/s]
Caching text embeddings to disk: 100%|##########| 47/47 [00:36<00:00, 1.30it/s] |
|
|
| Dataset: /app/ai-toolkit/datasets/images |
| - Preprocessing image dimensions |
|
0%| | 0/47 [00:00<?, ?it/s]
0%| | 0/47 [00:00<?, ?it/s]
100%|##########| 47/47 [00:00<00:00, 42799.02it/s]
100%|##########| 47/47 [00:00<00:00, 42799.02it/s] |
|
|
| - Found 47 images |
| Bucket sizes for /app/ai-toolkit/datasets/images: |
| 512x512: 39 files |
| 768x768: 7 files |
| 864x608: 1 files |
| 3 buckets made |
| Caching latents for /app/ai-toolkit/datasets/images |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching latents to disk: 21%|##1 | 10/47 [00:00<00:00, 63.39it/s]
Caching latents to disk: 21%|##1 | 10/47 [00:00<00:00, 63.39it/s]
Caching latents to disk: 36%|###6 | 17/47 [00:00<00:00, 53.41it/s]
Caching latents to disk: 36%|###6 | 17/47 [00:00<00:00, 53.41it/s]
Caching latents to disk: 49%|####8 | 23/47 [00:00<00:00, 47.57it/s]
Caching latents to disk: 49%|####8 | 23/47 [00:00<00:00, 47.57it/s]
Caching latents to disk: 60%|#####9 | 28/47 [00:00<00:00, 41.15it/s]
Caching latents to disk: 60%|#####9 | 28/47 [00:00<00:00, 41.15it/s]
Caching latents to disk: 74%|#######4 | 35/47 [00:00<00:00, 43.15it/s]
Caching latents to disk: 74%|#######4 | 35/47 [00:00<00:00, 43.15it/s]
Caching latents to disk: 85%|########5 | 40/47 [00:01<00:00, 29.03it/s]
Caching latents to disk: 85%|########5 | 40/47 [00:01<00:00, 29.03it/s]
Caching latents to disk: 98%|#########7| 46/47 [00:01<00:00, 31.90it/s]
Caching latents to disk: 98%|#########7| 46/47 [00:01<00:00, 31.90it/s]
Caching latents to disk: 100%|##########| 47/47 [00:01<00:00, 38.17it/s]
Caching latents to disk: 100%|##########| 47/47 [00:01<00:00, 38.17it/s] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/images |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/47 [00:00<?, ?it/s]
Caching text embeddings to disk: 100%|##########| 47/47 [00:00<00:00, 11633.66it/s]
Caching text embeddings to disk: 100%|##########| 47/47 [00:00<00:00, 11633.66it/s] |
|
|
| Caching embeddings and unloading text encoder |
|
|
| ***** UNLOADING TEXT ENCODER ***** |
| Embeddings cached to disk. We dont need the text encoder anymore |
| *********************************** |
|
|
|
|
