| Running 1 job |
| { |
| "type": "diffusion_trainer", |
| "training_folder": "/app/ai-toolkit/output", |
| "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", |
| "device": "cuda", |
| "trigger_word": null, |
| "performance_log_every": 10, |
| "network": { |
| "type": "lora", |
| "linear": 32, |
| "linear_alpha": 32, |
| "conv": 16, |
| "conv_alpha": 16, |
| "lokr_full_rank": true, |
| "lokr_factor": -1, |
| "network_kwargs": { |
| "ignore_if_contains": [] |
| } |
| }, |
| "save": { |
| "dtype": "bf16", |
| "save_every": 500, |
| "max_step_saves_to_keep": 400, |
| "save_format": "diffusers", |
| "push_to_hub": false |
| }, |
| "datasets": [ |
| { |
| "folder_path": "/app/ai-toolkit/datasets/49", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 49, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2, |
| "do_i2v": false, |
| "fps": 24 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/73", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 73, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/81", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 81, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/89", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 89, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/97", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 97, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 2 |
| }, |
| { |
| "folder_path": "/app/ai-toolkit/datasets/images", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0, |
| "cache_latents_to_disk": true, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 512, |
| 768 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 1, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 1 |
| } |
| ], |
| "train": { |
| "batch_size": 1, |
| "bypass_guidance_embedding": false, |
| "steps": 25000, |
| "gradient_accumulation": 1, |
| "train_unet": true, |
| "train_text_encoder": false, |
| "gradient_checkpointing": true, |
| "noise_scheduler": "flowmatch", |
| "optimizer": "adamw8bit", |
| "timestep_type": "weighted", |
| "content_or_style": "balanced", |
| "optimizer_params": { |
| "weight_decay": 0.0001 |
| }, |
| "unload_text_encoder": false, |
| "cache_text_embeddings": true, |
| "lr": 0.0001, |
| "ema_config": { |
| "use_ema": false, |
| "ema_decay": 0.99 |
| }, |
| "skip_first_sample": false, |
| "force_first_sample": false, |
| "disable_sampling": true, |
| "dtype": "bf16", |
| "diff_output_preservation": false, |
| "diff_output_preservation_multiplier": 1, |
| "diff_output_preservation_class": "person", |
| "switch_boundary_every": 1, |
| "loss_type": "mse" |
| }, |
| "logging": { |
| "log_every": 1, |
| "use_ui_logger": true |
| }, |
| "model": { |
| "name_or_path": "Lightricks/LTX-2", |
| "quantize": true, |
| "qtype": "qfloat8", |
| "quantize_te": true, |
| "qtype_te": "uint6", |
| "arch": "ltx2", |
| "low_vram": true, |
| "model_kwargs": {}, |
| "layer_offloading": false, |
| "layer_offloading_text_encoder_percent": 1, |
| "layer_offloading_transformer_percent": 1 |
| }, |
| "sample": { |
| "sampler": "flowmatch", |
| "sample_every": 250, |
| "width": 768, |
| "height": 768, |
| "samples": [], |
| "neg": "", |
| "seed": 42, |
| "walk_seed": true, |
| "guidance_scale": 4, |
| "sample_steps": 25, |
| "num_frames": 121, |
| "fps": 24 |
| } |
| } |
| Changing sample extention to animated webp |
| Using SQLite database at /app/ai-toolkit/aitk_db.db |
| Job ID: "bf530ed1-e796-42a1-9ffa-55399fe92986" |
|
|
| ############################################# |
| # Running job: PLORAV7_LTX |
| ############################################# |
|
|
|
|
| Running 1 process |
| Loading LTX2 model |
| Loading transformer |
|
config.json: 0.00B [00:00, ?B/s]
config.json: 0.00B [00:00, ?B/s]
config.json: 1.15kB [00:00, 8.14MB/s]
config.json: 1.15kB [00:00, 8.14MB/s] |
|
|
|
(…)ion_pytorch_model.safetensors.index.json: 0.00B [00:00, ?B/s]
(…)ion_pytorch_model.safetensors.index.json: 0.00B [00:00, ?B/s]
(…)ion_pytorch_model.safetensors.index.json: 378kB [00:00, 300MB/s]
(…)ion_pytorch_model.safetensors.index.json: 378kB [00:00, 300MB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 55.6k/4.99G [00:01<35:11:54, 39.3kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 55.6k/4.99G [00:01<35:11:54, 39.3kB/s]
transformer/diffusion_pytorch_model-0000(…): 22%|##2 | 1.10G/4.99G [00:01<00:04, 792MB/s]
transformer/diffusion_pytorch_model-0000(…): 22%|##2 | 1.10G/4.99G [00:01<00:04, 792MB/s]
transformer/diffusion_pytorch_model-0000(…): 43%|####3 | 2.15G/4.99G [00:01<00:01, 1.68GB/s]
transformer/diffusion_pytorch_model-0000(…): 43%|####3 | 2.15G/4.99G [00:01<00:01, 1.68GB/s]
transformer/diffusion_pytorch_model-0000(…): 54%|#####4 | 2.70G/4.99G [00:02<00:01, 1.58GB/s]
transformer/diffusion_pytorch_model-0000(…): 54%|#####4 | 2.70G/4.99G [00:02<00:01, 1.58GB/s]
transformer/diffusion_pytorch_model-0000(…): 65%|######4 | 3.22G/4.99G [00:02<00:00, 1.92GB/s]
transformer/diffusion_pytorch_model-0000(…): 65%|######4 | 3.22G/4.99G [00:02<00:00, 1.92GB/s]
transformer/diffusion_pytorch_model-0000(…): 74%|#######3 | 3.67G/4.99G [00:02<00:00, 2.24GB/s]
transformer/diffusion_pytorch_model-0000(…): 74%|#######3 | 3.67G/4.99G [00:02<00:00, 2.24GB/s]
transformer/diffusion_pytorch_model-0000(…): 82%|########1 | 4.08G/4.99G [00:02<00:00, 2.34GB/s]
transformer/diffusion_pytorch_model-0000(…): 82%|########1 | 4.08G/4.99G [00:02<00:00, 2.34GB/s]
transformer/diffusion_pytorch_model-0000(…): 90%|########9 | 4.47G/4.99G [00:02<00:00, 2.54GB/s]
transformer/diffusion_pytorch_model-0000(…): 90%|########9 | 4.47G/4.99G [00:02<00:00, 2.54GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:02<00:00, 1.72GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:02<00:00, 1.72GB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.97G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.97G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 52.2k/4.97G [00:00<24:08:14, 57.2kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 52.2k/4.97G [00:00<24:08:14, 57.2kB/s]
transformer/diffusion_pytorch_model-0000(…): 16%|#6 | 806M/4.97G [00:01<00:06, 614MB/s]
transformer/diffusion_pytorch_model-0000(…): 16%|#6 | 806M/4.97G [00:01<00:06, 614MB/s]
transformer/diffusion_pytorch_model-0000(…): 44%|####3 | 2.17G/4.97G [00:01<00:01, 1.48GB/s]
transformer/diffusion_pytorch_model-0000(…): 44%|####3 | 2.17G/4.97G [00:01<00:01, 1.48GB/s]
transformer/diffusion_pytorch_model-0000(…): 57%|#####6 | 2.82G/4.97G [00:02<00:01, 1.94GB/s]
transformer/diffusion_pytorch_model-0000(…): 57%|#####6 | 2.82G/4.97G [00:02<00:01, 1.94GB/s]
transformer/diffusion_pytorch_model-0000(…): 65%|######5 | 3.25G/4.97G [00:02<00:00, 2.15GB/s]
transformer/diffusion_pytorch_model-0000(…): 65%|######5 | 3.25G/4.97G [00:02<00:00, 2.15GB/s]
transformer/diffusion_pytorch_model-0000(…): 76%|#######6 | 3.79G/4.97G [00:02<00:00, 2.40GB/s]
transformer/diffusion_pytorch_model-0000(…): 76%|#######6 | 3.79G/4.97G [00:02<00:00, 2.40GB/s]
transformer/diffusion_pytorch_model-0000(…): 87%|########6 | 4.30G/4.97G [00:02<00:00, 2.64GB/s]
transformer/diffusion_pytorch_model-0000(…): 87%|########6 | 4.30G/4.97G [00:02<00:00, 2.64GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.97G/4.97G [00:02<00:00, 3.12GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.97G/4.97G [00:02<00:00, 3.12GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.97G/4.97G [00:02<00:00, 1.86GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.97G/4.97G [00:02<00:00, 1.86GB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 22.3k/4.99G [00:00<57:27:46, 24.1kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 22.3k/4.99G [00:00<57:27:46, 24.1kB/s]
transformer/diffusion_pytorch_model-0000(…): 12%|#2 | 604M/4.99G [00:01<00:08, 535MB/s]
transformer/diffusion_pytorch_model-0000(…): 12%|#2 | 604M/4.99G [00:01<00:08, 535MB/s]
transformer/diffusion_pytorch_model-0000(…): 40%|#### | 2.01G/4.99G [00:01<00:01, 1.57GB/s]
transformer/diffusion_pytorch_model-0000(…): 40%|#### | 2.01G/4.99G [00:01<00:01, 1.57GB/s]
transformer/diffusion_pytorch_model-0000(…): 58%|#####8 | 2.92G/4.99G [00:01<00:00, 2.27GB/s]
transformer/diffusion_pytorch_model-0000(…): 58%|#####8 | 2.92G/4.99G [00:01<00:00, 2.27GB/s]
transformer/diffusion_pytorch_model-0000(…): 66%|######6 | 3.32G/4.99G [00:02<00:00, 2.50GB/s]
transformer/diffusion_pytorch_model-0000(…): 66%|######6 | 3.32G/4.99G [00:02<00:00, 2.50GB/s]
transformer/diffusion_pytorch_model-0000(…): 75%|#######4 | 3.74G/4.99G [00:02<00:00, 2.64GB/s]
transformer/diffusion_pytorch_model-0000(…): 75%|#######4 | 3.74G/4.99G [00:02<00:00, 2.64GB/s]
transformer/diffusion_pytorch_model-0000(…): 83%|########2 | 4.12G/4.99G [00:02<00:00, 2.60GB/s]
transformer/diffusion_pytorch_model-0000(…): 83%|########2 | 4.12G/4.99G [00:02<00:00, 2.60GB/s]
transformer/diffusion_pytorch_model-0000(…): 89%|########9 | 4.46G/4.99G [00:02<00:00, 2.68GB/s]
transformer/diffusion_pytorch_model-0000(…): 89%|########9 | 4.46G/4.99G [00:02<00:00, 2.68GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:02<00:00, 2.01GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:02<00:00, 2.01GB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.98G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.98G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 434k/4.98G [00:00<3:06:19, 445kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 434k/4.98G [00:00<3:06:19, 445kB/s]
transformer/diffusion_pytorch_model-0000(…): 22%|##1 | 1.07G/4.98G [00:01<00:04, 861MB/s]
transformer/diffusion_pytorch_model-0000(…): 22%|##1 | 1.07G/4.98G [00:01<00:04, 861MB/s]
transformer/diffusion_pytorch_model-0000(…): 43%|####3 | 2.15G/4.98G [00:01<00:01, 1.42GB/s]
transformer/diffusion_pytorch_model-0000(…): 43%|####3 | 2.15G/4.98G [00:01<00:01, 1.42GB/s]
transformer/diffusion_pytorch_model-0000(…): 55%|#####5 | 2.75G/4.98G [00:02<00:01, 1.86GB/s]
transformer/diffusion_pytorch_model-0000(…): 55%|#####5 | 2.75G/4.98G [00:02<00:01, 1.86GB/s]
transformer/diffusion_pytorch_model-0000(…): 62%|######2 | 3.09G/4.98G [00:02<00:00, 2.04GB/s]
transformer/diffusion_pytorch_model-0000(…): 62%|######2 | 3.09G/4.98G [00:02<00:00, 2.04GB/s]
transformer/diffusion_pytorch_model-0000(…): 69%|######9 | 3.45G/4.98G [00:02<00:00, 2.11GB/s]
transformer/diffusion_pytorch_model-0000(…): 69%|######9 | 3.45G/4.98G [00:02<00:00, 2.11GB/s]
transformer/diffusion_pytorch_model-0000(…): 77%|#######7 | 3.84G/4.98G [00:02<00:00, 2.33GB/s]
transformer/diffusion_pytorch_model-0000(…): 77%|#######7 | 3.84G/4.98G [00:02<00:00, 2.33GB/s]
transformer/diffusion_pytorch_model-0000(…): 84%|########3 | 4.17G/4.98G [00:02<00:00, 2.38GB/s]
transformer/diffusion_pytorch_model-0000(…): 84%|########3 | 4.17G/4.98G [00:02<00:00, 2.38GB/s]
transformer/diffusion_pytorch_model-0000(…): 99%|#########8| 4.91G/4.98G [00:02<00:00, 3.34GB/s]
transformer/diffusion_pytorch_model-0000(…): 99%|#########8| 4.91G/4.98G [00:02<00:00, 3.34GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.98G/4.98G [00:02<00:00, 1.86GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.98G/4.98G [00:02<00:00, 1.86GB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.97G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.97G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 16.5k/4.97G [00:00<81:23:50, 17.0kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 16.5k/4.97G [00:00<81:23:50, 17.0kB/s]
transformer/diffusion_pytorch_model-0000(…): 16%|#6 | 805M/4.97G [00:01<00:06, 684MB/s]
transformer/diffusion_pytorch_model-0000(…): 16%|#6 | 805M/4.97G [00:01<00:06, 684MB/s]
transformer/diffusion_pytorch_model-0000(…): 42%|####1 | 2.08G/4.97G [00:01<00:01, 1.49GB/s]
transformer/diffusion_pytorch_model-0000(…): 42%|####1 | 2.08G/4.97G [00:01<00:01, 1.49GB/s]
transformer/diffusion_pytorch_model-0000(…): 57%|#####6 | 2.82G/4.97G [00:02<00:01, 1.94GB/s]
transformer/diffusion_pytorch_model-0000(…): 57%|#####6 | 2.82G/4.97G [00:02<00:01, 1.94GB/s]
transformer/diffusion_pytorch_model-0000(…): 65%|######4 | 3.22G/4.97G [00:02<00:00, 2.15GB/s]
transformer/diffusion_pytorch_model-0000(…): 65%|######4 | 3.22G/4.97G [00:02<00:00, 2.15GB/s]
transformer/diffusion_pytorch_model-0000(…): 74%|#######4 | 3.69G/4.97G [00:02<00:00, 2.46GB/s]
transformer/diffusion_pytorch_model-0000(…): 74%|#######4 | 3.69G/4.97G [00:02<00:00, 2.46GB/s]
transformer/diffusion_pytorch_model-0000(…): 82%|########1 | 4.06G/4.97G [00:02<00:00, 2.50GB/s]
transformer/diffusion_pytorch_model-0000(…): 82%|########1 | 4.06G/4.97G [00:02<00:00, 2.50GB/s]
transformer/diffusion_pytorch_model-0000(…): 89%|########9 | 4.43G/4.97G [00:02<00:00, 2.71GB/s]
transformer/diffusion_pytorch_model-0000(…): 89%|########9 | 4.43G/4.97G [00:02<00:00, 2.71GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.97G/4.97G [00:02<00:00, 1.91GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.97G/4.97G [00:02<00:00, 1.91GB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.94G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.94G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 34.3k/4.94G [00:00<36:00:25, 38.1kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 34.3k/4.94G [00:00<36:00:25, 38.1kB/s]
transformer/diffusion_pytorch_model-0000(…): 15%|#4 | 738M/4.94G [00:01<00:06, 660MB/s]
transformer/diffusion_pytorch_model-0000(…): 15%|#4 | 738M/4.94G [00:01<00:06, 660MB/s]
transformer/diffusion_pytorch_model-0000(…): 39%|###9 | 1.95G/4.94G [00:01<00:01, 1.55GB/s]
transformer/diffusion_pytorch_model-0000(…): 39%|###9 | 1.95G/4.94G [00:01<00:01, 1.55GB/s]
transformer/diffusion_pytorch_model-0000(…): 53%|#####2 | 2.60G/4.94G [00:01<00:01, 2.02GB/s]
transformer/diffusion_pytorch_model-0000(…): 53%|#####2 | 2.60G/4.94G [00:01<00:01, 2.02GB/s]
transformer/diffusion_pytorch_model-0000(…): 59%|#####9 | 2.91G/4.94G [00:01<00:00, 2.06GB/s]
transformer/diffusion_pytorch_model-0000(…): 59%|#####9 | 2.91G/4.94G [00:01<00:00, 2.06GB/s]
transformer/diffusion_pytorch_model-0000(…): 69%|######8 | 3.38G/4.94G [00:02<00:00, 2.44GB/s]
transformer/diffusion_pytorch_model-0000(…): 69%|######8 | 3.38G/4.94G [00:02<00:00, 2.44GB/s]
transformer/diffusion_pytorch_model-0000(…): 76%|#######5 | 3.75G/4.94G [00:02<00:00, 2.55GB/s]
transformer/diffusion_pytorch_model-0000(…): 76%|#######5 | 3.75G/4.94G [00:02<00:00, 2.55GB/s]
transformer/diffusion_pytorch_model-0000(…): 84%|########3 | 4.13G/4.94G [00:02<00:00, 2.12GB/s]
transformer/diffusion_pytorch_model-0000(…): 84%|########3 | 4.13G/4.94G [00:02<00:00, 2.12GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.94G/4.94G [00:02<00:00, 1.93GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.94G/4.94G [00:02<00:00, 1.93GB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 98.3k/4.99G [00:00<13:22:46, 104kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 98.3k/4.99G [00:00<13:22:46, 104kB/s]
transformer/diffusion_pytorch_model-0000(…): 8%|8 | 416M/4.99G [00:01<00:13, 346MB/s]
transformer/diffusion_pytorch_model-0000(…): 8%|8 | 416M/4.99G [00:01<00:13, 346MB/s]
transformer/diffusion_pytorch_model-0000(…): 29%|##8 | 1.44G/4.99G [00:01<00:03, 1.07GB/s]
transformer/diffusion_pytorch_model-0000(…): 29%|##8 | 1.44G/4.99G [00:01<00:03, 1.07GB/s]
transformer/diffusion_pytorch_model-0000(…): 43%|####2 | 2.15G/4.99G [00:01<00:01, 1.65GB/s]
transformer/diffusion_pytorch_model-0000(…): 43%|####2 | 2.15G/4.99G [00:01<00:01, 1.65GB/s]
transformer/diffusion_pytorch_model-0000(…): 50%|####9 | 2.48G/4.99G [00:02<00:01, 1.65GB/s]
transformer/diffusion_pytorch_model-0000(…): 50%|####9 | 2.48G/4.99G [00:02<00:01, 1.65GB/s]
transformer/diffusion_pytorch_model-0000(…): 56%|#####5 | 2.79G/4.99G [00:02<00:01, 1.83GB/s]
transformer/diffusion_pytorch_model-0000(…): 56%|#####5 | 2.79G/4.99G [00:02<00:01, 1.83GB/s]
transformer/diffusion_pytorch_model-0000(…): 61%|######1 | 3.06G/4.99G [00:02<00:01, 1.92GB/s]
transformer/diffusion_pytorch_model-0000(…): 61%|######1 | 3.06G/4.99G [00:02<00:01, 1.92GB/s]
transformer/diffusion_pytorch_model-0000(…): 67%|######6 | 3.32G/4.99G [00:02<00:00, 2.02GB/s]
transformer/diffusion_pytorch_model-0000(…): 67%|######6 | 3.32G/4.99G [00:02<00:00, 2.02GB/s]
transformer/diffusion_pytorch_model-0000(…): 77%|#######7 | 3.86G/4.99G [00:02<00:00, 2.67GB/s]
transformer/diffusion_pytorch_model-0000(…): 77%|#######7 | 3.86G/4.99G [00:02<00:00, 2.67GB/s]
transformer/diffusion_pytorch_model-0000(…): 84%|########3 | 4.19G/4.99G [00:02<00:00, 2.35GB/s]
transformer/diffusion_pytorch_model-0000(…): 84%|########3 | 4.19G/4.99G [00:02<00:00, 2.35GB/s]
transformer/diffusion_pytorch_model-0000(…): 91%|######### | 4.53G/4.99G [00:04<00:00, 687MB/s]
transformer/diffusion_pytorch_model-0000(…): 91%|######### | 4.53G/4.99G [00:04<00:00, 687MB/s]
transformer/diffusion_pytorch_model-0000(…): 96%|#########5| 4.79G/4.99G [00:04<00:00, 603MB/s]
transformer/diffusion_pytorch_model-0000(…): 96%|#########5| 4.79G/4.99G [00:04<00:00, 603MB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:05<00:00, 616MB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:05<00:00, 616MB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:05<00:00, 973MB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 4.99G/4.99G [00:05<00:00, 973MB/s] |
|
|
|
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/2.93G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 0.00/2.93G [00:00<?, ?B/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 255k/2.93G [00:00<3:10:08, 257kB/s]
transformer/diffusion_pytorch_model-0000(…): 0%| | 255k/2.93G [00:00<3:10:08, 257kB/s]
transformer/diffusion_pytorch_model-0000(…): 14%|#3 | 403M/2.93G [00:01<00:06, 387MB/s]
transformer/diffusion_pytorch_model-0000(…): 14%|#3 | 403M/2.93G [00:01<00:06, 387MB/s]
transformer/diffusion_pytorch_model-0000(…): 37%|###6 | 1.07G/2.93G [00:01<00:01, 1.14GB/s]
transformer/diffusion_pytorch_model-0000(…): 37%|###6 | 1.07G/2.93G [00:01<00:01, 1.14GB/s]
transformer/diffusion_pytorch_model-0000(…): 50%|####9 | 1.46G/2.93G [00:01<00:01, 1.30GB/s]
transformer/diffusion_pytorch_model-0000(…): 50%|####9 | 1.46G/2.93G [00:01<00:01, 1.30GB/s]
transformer/diffusion_pytorch_model-0000(…): 59%|#####8 | 1.72G/2.93G [00:01<00:00, 1.28GB/s]
transformer/diffusion_pytorch_model-0000(…): 59%|#####8 | 1.72G/2.93G [00:01<00:00, 1.28GB/s]
transformer/diffusion_pytorch_model-0000(…): 76%|#######5 | 2.22G/2.93G [00:02<00:00, 1.75GB/s]
transformer/diffusion_pytorch_model-0000(…): 76%|#######5 | 2.22G/2.93G [00:02<00:00, 1.75GB/s]
transformer/diffusion_pytorch_model-0000(…): 86%|########6 | 2.53G/2.93G [00:02<00:00, 1.53GB/s]
transformer/diffusion_pytorch_model-0000(…): 86%|########6 | 2.53G/2.93G [00:02<00:00, 1.53GB/s]
transformer/diffusion_pytorch_model-0000(…): 95%|#########5| 2.80G/2.93G [00:02<00:00, 1.07GB/s]
transformer/diffusion_pytorch_model-0000(…): 95%|#########5| 2.80G/2.93G [00:02<00:00, 1.07GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 2.93G/2.93G [00:02<00:00, 1.05GB/s]
transformer/diffusion_pytorch_model-0000(…): 100%|##########| 2.93G/2.93G [00:02<00:00, 1.05GB/s] |
|
|
|
Loading checkpoint shards: 0%| | 0/8 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/8 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|##5 | 2/8 [00:00<00:00, 16.24it/s]
Loading checkpoint shards: 25%|##5 | 2/8 [00:00<00:00, 16.24it/s]
Loading checkpoint shards: 75%|#######5 | 6/8 [00:00<00:00, 28.15it/s]
Loading checkpoint shards: 75%|#######5 | 6/8 [00:00<00:00, 28.15it/s]
Loading checkpoint shards: 100%|##########| 8/8 [00:00<00:00, 29.10it/s]
Loading checkpoint shards: 100%|##########| 8/8 [00:00<00:00, 29.10it/s] |
|
|
| Quantizing Transformer |
| - quantizing 48 transformer blocks |
|
0%| | 0/48 [00:00<?, ?it/s]
0%| | 0/48 [00:00<?, ?it/s]
2%|2 | 1/48 [00:00<00:27, 1.70it/s]
2%|2 | 1/48 [00:00<00:27, 1.70it/s]
4%|4 | 2/48 [00:00<00:19, 2.35it/s]
4%|4 | 2/48 [00:00<00:19, 2.35it/s]
6%|6 | 3/48 [00:01<00:18, 2.43it/s]
6%|6 | 3/48 [00:01<00:18, 2.43it/s]
8%|8 | 4/48 [00:01<00:16, 2.66it/s]
8%|8 | 4/48 [00:01<00:16, 2.66it/s]
10%|# | 5/48 [00:01<00:15, 2.82it/s]
10%|# | 5/48 [00:01<00:15, 2.82it/s]
12%|#2 | 6/48 [00:02<00:16, 2.62it/s]
12%|#2 | 6/48 [00:02<00:16, 2.62it/s]
15%|#4 | 7/48 [00:02<00:16, 2.47it/s]
15%|#4 | 7/48 [00:02<00:16, 2.47it/s]
17%|#6 | 8/48 [00:03<00:16, 2.45it/s]
17%|#6 | 8/48 [00:03<00:16, 2.45it/s]
19%|#8 | 9/48 [00:03<00:16, 2.41it/s]
19%|#8 | 9/48 [00:03<00:16, 2.41it/s]
21%|## | 10/48 [00:04<00:14, 2.54it/s]
21%|## | 10/48 [00:04<00:14, 2.54it/s]
23%|##2 | 11/48 [00:04<00:15, 2.38it/s]
23%|##2 | 11/48 [00:04<00:15, 2.38it/s]
25%|##5 | 12/48 [00:04<00:15, 2.29it/s]
25%|##5 | 12/48 [00:04<00:15, 2.29it/s]
27%|##7 | 13/48 [00:05<00:16, 2.19it/s]
27%|##7 | 13/48 [00:05<00:16, 2.19it/s]
29%|##9 | 14/48 [00:05<00:14, 2.40it/s]
29%|##9 | 14/48 [00:05<00:14, 2.40it/s]
31%|###1 | 15/48 [00:06<00:12, 2.55it/s]
31%|###1 | 15/48 [00:06<00:12, 2.55it/s]
33%|###3 | 16/48 [00:06<00:11, 2.68it/s]
33%|###3 | 16/48 [00:06<00:11, 2.68it/s]
35%|###5 | 17/48 [00:06<00:11, 2.80it/s]
35%|###5 | 17/48 [00:06<00:11, 2.80it/s]
38%|###7 | 18/48 [00:07<00:10, 2.87it/s]
38%|###7 | 18/48 [00:07<00:10, 2.87it/s]
40%|###9 | 19/48 [00:07<00:11, 2.56it/s]
40%|###9 | 19/48 [00:07<00:11, 2.56it/s]
42%|####1 | 20/48 [00:07<00:10, 2.70it/s]
42%|####1 | 20/48 [00:07<00:10, 2.70it/s]
44%|####3 | 21/48 [00:08<00:09, 2.79it/s]
44%|####3 | 21/48 [00:08<00:09, 2.79it/s]
46%|####5 | 22/48 [00:08<00:09, 2.86it/s]
46%|####5 | 22/48 [00:08<00:09, 2.86it/s]
48%|####7 | 23/48 [00:08<00:08, 2.92it/s]
48%|####7 | 23/48 [00:08<00:08, 2.92it/s]
50%|##### | 24/48 [00:09<00:08, 2.96it/s]
50%|##### | 24/48 [00:09<00:08, 2.96it/s]
52%|#####2 | 25/48 [00:09<00:08, 2.62it/s]
52%|#####2 | 25/48 [00:09<00:08, 2.62it/s]
54%|#####4 | 26/48 [00:10<00:08, 2.74it/s]
54%|#####4 | 26/48 [00:10<00:08, 2.74it/s]
56%|#####6 | 27/48 [00:10<00:07, 2.84it/s]
56%|#####6 | 27/48 [00:10<00:07, 2.84it/s]
58%|#####8 | 28/48 [00:10<00:06, 2.89it/s]
58%|#####8 | 28/48 [00:10<00:06, 2.89it/s]
60%|###### | 29/48 [00:11<00:06, 2.96it/s]
60%|###### | 29/48 [00:11<00:06, 2.96it/s]
62%|######2 | 30/48 [00:11<00:05, 3.01it/s]
62%|######2 | 30/48 [00:11<00:05, 3.01it/s]
65%|######4 | 31/48 [00:11<00:05, 3.01it/s]
65%|######4 | 31/48 [00:11<00:05, 3.01it/s]
67%|######6 | 32/48 [00:12<00:06, 2.65it/s]
67%|######6 | 32/48 [00:12<00:06, 2.65it/s]
69%|######8 | 33/48 [00:12<00:05, 2.77it/s]
69%|######8 | 33/48 [00:12<00:05, 2.77it/s]
71%|####### | 34/48 [00:12<00:04, 2.85it/s]
71%|####### | 34/48 [00:12<00:04, 2.85it/s]
73%|#######2 | 35/48 [00:13<00:04, 2.90it/s]
73%|#######2 | 35/48 [00:13<00:04, 2.90it/s]
75%|#######5 | 36/48 [00:13<00:04, 2.95it/s]
75%|#######5 | 36/48 [00:13<00:04, 2.95it/s]
77%|#######7 | 37/48 [00:13<00:03, 2.97it/s]
77%|#######7 | 37/48 [00:13<00:03, 2.97it/s]
79%|#######9 | 38/48 [00:14<00:03, 2.61it/s]
79%|#######9 | 38/48 [00:14<00:03, 2.61it/s]
81%|########1 | 39/48 [00:14<00:03, 2.73it/s]
81%|########1 | 39/48 [00:14<00:03, 2.73it/s]
83%|########3 | 40/48 [00:14<00:02, 2.82it/s]
83%|########3 | 40/48 [00:14<00:02, 2.82it/s]
85%|########5 | 41/48 [00:15<00:02, 2.90it/s]
85%|########5 | 41/48 [00:15<00:02, 2.90it/s]
88%|########7 | 42/48 [00:15<00:02, 2.95it/s]
88%|########7 | 42/48 [00:15<00:02, 2.95it/s]
90%|########9 | 43/48 [00:15<00:01, 2.99it/s]
90%|########9 | 43/48 [00:15<00:01, 2.99it/s]
92%|#########1| 44/48 [00:16<00:01, 3.01it/s]
92%|#########1| 44/48 [00:16<00:01, 3.01it/s]
94%|#########3| 45/48 [00:16<00:01, 2.63it/s]
94%|#########3| 45/48 [00:16<00:01, 2.63it/s]
96%|#########5| 46/48 [00:17<00:00, 2.75it/s]
96%|#########5| 46/48 [00:17<00:00, 2.75it/s]
98%|#########7| 47/48 [00:17<00:00, 2.85it/s]
98%|#########7| 47/48 [00:17<00:00, 2.85it/s]
100%|##########| 48/48 [00:17<00:00, 2.91it/s]
100%|##########| 48/48 [00:17<00:00, 2.91it/s]
100%|##########| 48/48 [00:17<00:00, 2.71it/s]
100%|##########| 48/48 [00:17<00:00, 2.71it/s] |
|
|
| - quantizing extras |
| Moving transformer to CPU |
| Loading text encoder |
|
tokenizer_config.json: 0.00B [00:00, ?B/s]
tokenizer_config.json: 0.00B [00:00, ?B/s]
tokenizer_config.json: 1.16MB [00:00, 304MB/s]
tokenizer_config.json: 1.16MB [00:00, 304MB/s] |
|
|
|
tokenizer/tokenizer.model: 0%| | 0.00/4.69M [00:00<?, ?B/s]
tokenizer/tokenizer.model: 0%| | 0.00/4.69M [00:00<?, ?B/s]
tokenizer/tokenizer.model: 100%|##########| 4.69M/4.69M [00:00<00:00, 13.5MB/s]
tokenizer/tokenizer.model: 100%|##########| 4.69M/4.69M [00:00<00:00, 13.5MB/s]
tokenizer/tokenizer.model: 100%|##########| 4.69M/4.69M [00:00<00:00, 13.5MB/s]
tokenizer/tokenizer.model: 100%|##########| 4.69M/4.69M [00:00<00:00, 13.5MB/s] |
|
|
|
tokenizer/tokenizer.json: 0%| | 0.00/33.4M [00:00<?, ?B/s]
tokenizer/tokenizer.json: 0%| | 0.00/33.4M [00:00<?, ?B/s]
tokenizer/tokenizer.json: 100%|##########| 33.4M/33.4M [00:00<00:00, 157MB/s]
tokenizer/tokenizer.json: 100%|##########| 33.4M/33.4M [00:00<00:00, 157MB/s]
tokenizer/tokenizer.json: 100%|##########| 33.4M/33.4M [00:00<00:00, 156MB/s]
tokenizer/tokenizer.json: 100%|##########| 33.4M/33.4M [00:00<00:00, 156MB/s] |
|
|
|
added_tokens.json: 0%| | 0.00/35.0 [00:00<?, ?B/s]
added_tokens.json: 0%| | 0.00/35.0 [00:00<?, ?B/s]
added_tokens.json: 100%|##########| 35.0/35.0 [00:00<00:00, 188kB/s]
added_tokens.json: 100%|##########| 35.0/35.0 [00:00<00:00, 188kB/s] |
|
|
|
special_tokens_map.json: 0%| | 0.00/662 [00:00<?, ?B/s]
special_tokens_map.json: 0%| | 0.00/662 [00:00<?, ?B/s]
special_tokens_map.json: 100%|##########| 662/662 [00:00<00:00, 3.01MB/s]
special_tokens_map.json: 100%|##########| 662/662 [00:00<00:00, 3.01MB/s] |
|
|
|
chat_template.jinja: 0.00B [00:00, ?B/s]
chat_template.jinja: 0.00B [00:00, ?B/s]
chat_template.jinja: 1.53kB [00:00, 3.83MB/s]
chat_template.jinja: 1.53kB [00:00, 3.83MB/s] |
|
|
|
config.json: 0.00B [00:00, ?B/s]
config.json: 0.00B [00:00, ?B/s]
config.json: 2.96kB [00:00, 15.9MB/s]
config.json: 2.96kB [00:00, 15.9MB/s] |
|
|
|
model.safetensors.index.json: 0.00B [00:00, ?B/s]
model.safetensors.index.json: 0.00B [00:00, ?B/s]
model.safetensors.index.json: 109kB [00:00, 117MB/s]
model.safetensors.index.json: 109kB [00:00, 117MB/s] |
|
|
|
text_encoder/model-00001-of-00011.safete(…): 0%| | 0.00/1.69G [00:00<?, ?B/s]
text_encoder/model-00001-of-00011.safete(…): 0%| | 0.00/1.69G [00:00<?, ?B/s]
text_encoder/model-00001-of-00011.safete(…): 0%| | 2.97M/1.69G [00:00<08:35, 3.26MB/s]
text_encoder/model-00001-of-00011.safete(…): 0%| | 2.97M/1.69G [00:00<08:35, 3.26MB/s]
text_encoder/model-00001-of-00011.safete(…): 41%|#### | 683M/1.69G [00:01<00:01, 869MB/s]
text_encoder/model-00001-of-00011.safete(…): 41%|#### | 683M/1.69G [00:01<00:01, 869MB/s]
text_encoder/model-00001-of-00011.safete(…): 68%|######8 | 1.15G/1.69G [00:01<00:00, 1.38GB/s]
text_encoder/model-00001-of-00011.safete(…): 68%|######8 | 1.15G/1.69G [00:01<00:00, 1.38GB/s]
text_encoder/model-00001-of-00011.safete(…): 88%|########8 | 1.48G/1.69G [00:01<00:00, 1.71GB/s]
text_encoder/model-00001-of-00011.safete(…): 88%|########8 | 1.48G/1.69G [00:01<00:00, 1.71GB/s]
text_encoder/model-00001-of-00011.safete(…): 100%|##########| 1.69G/1.69G [00:01<00:00, 1.14GB/s]
text_encoder/model-00001-of-00011.safete(…): 100%|##########| 1.69G/1.69G [00:01<00:00, 1.14GB/s] |
|
|
|
text_encoder/model-00002-of-00011.safete(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
text_encoder/model-00002-of-00011.safete(…): 0%| | 0.00/4.99G [00:00<?, ?B/s]
text_encoder/model-00002-of-00011.safete(…): 0%| | 203k/4.99G [00:00<3:54:02, 355kB/s]
text_encoder/model-00002-of-00011.safete(…): 0%| | 203k/4.99G [00:00<3:54:02, 355kB/s]
text_encoder/model-00002-of-00011.safete(…): 7%|6 | 329M/4.99G [00:00<00:07, 657MB/s]
text_encoder/model-00002-of-00011.safete(…): 7%|6 | 329M/4.99G [00:00<00:07, 657MB/s]
text_encoder/model-00002-of-00011.safete(…): 11%|#1 | 559M/4.99G [00:00<00:04, 1.02GB/s]
text_encoder/model-00002-of-00011.safete(…): 11%|#1 | 559M/4.99G [00:00<00:04, 1.02GB/s]
text_encoder/model-00002-of-00011.safete(…): 15%|#5 | 770M/4.99G [00:01<00:07, 531MB/s]
text_encoder/model-00002-of-00011.safete(…): 15%|#5 | 770M/4.99G [00:01<00:07, 531MB/s]
text_encoder/model-00002-of-00011.safete(…): 27%|##6 | 1.34G/4.99G [00:01<00:03, 1.17GB/s]
text_encoder/model-00002-of-00011.safete(…): 27%|##6 | 1.34G/4.99G [00:01<00:03, 1.17GB/s]
text_encoder/model-00002-of-00011.safete(…): 35%|###4 | 1.73G/4.99G [00:01<00:02, 1.57GB/s]
text_encoder/model-00002-of-00011.safete(…): 35%|###4 | 1.73G/4.99G [00:01<00:02, 1.57GB/s]
text_encoder/model-00002-of-00011.safete(…): 42%|####2 | 2.12G/4.99G [00:01<00:01, 1.99GB/s]
text_encoder/model-00002-of-00011.safete(…): 42%|####2 | 2.12G/4.99G [00:01<00:01, 1.99GB/s]
text_encoder/model-00002-of-00011.safete(…): 50%|##### | 2.50G/4.99G [00:01<00:01, 2.36GB/s]
text_encoder/model-00002-of-00011.safete(…): 50%|##### | 2.50G/4.99G [00:01<00:01, 2.36GB/s]
text_encoder/model-00002-of-00011.safete(…): 57%|#####7 | 2.87G/4.99G [00:01<00:00, 2.57GB/s]
text_encoder/model-00002-of-00011.safete(…): 57%|#####7 | 2.87G/4.99G [00:01<00:00, 2.57GB/s]
text_encoder/model-00002-of-00011.safete(…): 64%|######4 | 3.21G/4.99G [00:02<00:00, 2.70GB/s]
text_encoder/model-00002-of-00011.safete(…): 64%|######4 | 3.21G/4.99G [00:02<00:00, 2.70GB/s]
text_encoder/model-00002-of-00011.safete(…): 71%|#######1 | 3.55G/4.99G [00:02<00:00, 2.68GB/s]
text_encoder/model-00002-of-00011.safete(…): 71%|#######1 | 3.55G/4.99G [00:02<00:00, 2.68GB/s]
text_encoder/model-00002-of-00011.safete(…): 78%|#######7 | 3.88G/4.99G [00:02<00:00, 2.56GB/s]
text_encoder/model-00002-of-00011.safete(…): 78%|#######7 | 3.88G/4.99G [00:02<00:00, 2.56GB/s]
text_encoder/model-00002-of-00011.safete(…): 85%|########5 | 4.26G/4.99G [00:02<00:00, 2.50GB/s]
text_encoder/model-00002-of-00011.safete(…): 85%|########5 | 4.26G/4.99G [00:02<00:00, 2.50GB/s]
text_encoder/model-00002-of-00011.safete(…): 92%|#########1| 4.58G/4.99G [00:02<00:00, 2.49GB/s]
text_encoder/model-00002-of-00011.safete(…): 92%|#########1| 4.58G/4.99G [00:02<00:00, 2.49GB/s]
text_encoder/model-00002-of-00011.safete(…): 100%|##########| 4.99G/4.99G [00:02<00:00, 1.84GB/s]
text_encoder/model-00002-of-00011.safete(…): 100%|##########| 4.99G/4.99G [00:02<00:00, 1.84GB/s] |
|
|
|
text_encoder/model-00003-of-00011.safete(…): 0%| | 0.00/4.84G [00:00<?, ?B/s]
text_encoder/model-00003-of-00011.safete(…): 0%| | 0.00/4.84G [00:00<?, ?B/s]
text_encoder/model-00003-of-00011.safete(…): 0%| | 23.2k/4.84G [00:00<57:35:27, 23.4kB/s]
text_encoder/model-00003-of-00011.safete(…): 0%| | 23.2k/4.84G [00:00<57:35:27, 23.4kB/s]
text_encoder/model-00003-of-00011.safete(…): 17%|#6 | 805M/4.84G [00:01<00:05, 678MB/s]
text_encoder/model-00003-of-00011.safete(…): 17%|#6 | 805M/4.84G [00:01<00:05, 678MB/s]
text_encoder/model-00003-of-00011.safete(…): 46%|####5 | 2.21G/4.84G [00:01<00:01, 1.83GB/s]
text_encoder/model-00003-of-00011.safete(…): 46%|####5 | 2.21G/4.84G [00:01<00:01, 1.83GB/s]
text_encoder/model-00003-of-00011.safete(…): 54%|#####4 | 2.62G/4.84G [00:01<00:01, 2.06GB/s]
text_encoder/model-00003-of-00011.safete(…): 54%|#####4 | 2.62G/4.84G [00:01<00:01, 2.06GB/s]
text_encoder/model-00003-of-00011.safete(…): 61%|###### | 2.95G/4.84G [00:02<00:01, 1.74GB/s]
text_encoder/model-00003-of-00011.safete(…): 61%|###### | 2.95G/4.84G [00:02<00:01, 1.74GB/s]
text_encoder/model-00003-of-00011.safete(…): 74%|#######3 | 3.57G/4.84G [00:02<00:00, 2.25GB/s]
text_encoder/model-00003-of-00011.safete(…): 74%|#######3 | 3.57G/4.84G [00:02<00:00, 2.25GB/s]
text_encoder/model-00003-of-00011.safete(…): 81%|######## | 3.91G/4.84G [00:02<00:00, 2.17GB/s]
text_encoder/model-00003-of-00011.safete(…): 81%|######## | 3.91G/4.84G [00:02<00:00, 2.17GB/s]
text_encoder/model-00003-of-00011.safete(…): 97%|#########7| 4.71G/4.84G [00:02<00:00, 2.54GB/s]
text_encoder/model-00003-of-00011.safete(…): 97%|#########7| 4.71G/4.84G [00:02<00:00, 2.54GB/s]
text_encoder/model-00003-of-00011.safete(…): 100%|##########| 4.84G/4.84G [00:02<00:00, 1.78GB/s]
text_encoder/model-00003-of-00011.safete(…): 100%|##########| 4.84G/4.84G [00:02<00:00, 1.78GB/s] |
|
|
|
text_encoder/model-00004-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00004-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00004-of-00011.safete(…): 0%| | 185k/4.95G [00:00<7:09:28, 192kB/s]
text_encoder/model-00004-of-00011.safete(…): 0%| | 185k/4.95G [00:00<7:09:28, 192kB/s]
text_encoder/model-00004-of-00011.safete(…): 19%|#8 | 941M/4.95G [00:01<00:04, 827MB/s]
text_encoder/model-00004-of-00011.safete(…): 19%|#8 | 941M/4.95G [00:01<00:04, 827MB/s]
text_encoder/model-00004-of-00011.safete(…): 37%|###6 | 1.81G/4.95G [00:01<00:02, 1.21GB/s]
text_encoder/model-00004-of-00011.safete(…): 37%|###6 | 1.81G/4.95G [00:01<00:02, 1.21GB/s]
text_encoder/model-00004-of-00011.safete(…): 58%|#####8 | 2.89G/4.95G [00:02<00:00, 2.09GB/s]
text_encoder/model-00004-of-00011.safete(…): 58%|#####8 | 2.89G/4.95G [00:02<00:00, 2.09GB/s]
text_encoder/model-00004-of-00011.safete(…): 74%|#######4 | 3.69G/4.95G [00:02<00:00, 2.64GB/s]
text_encoder/model-00004-of-00011.safete(…): 74%|#######4 | 3.69G/4.95G [00:02<00:00, 2.64GB/s]
text_encoder/model-00004-of-00011.safete(…): 84%|########3 | 4.16G/4.95G [00:02<00:00, 2.41GB/s]
text_encoder/model-00004-of-00011.safete(…): 84%|########3 | 4.16G/4.95G [00:02<00:00, 2.41GB/s]
text_encoder/model-00004-of-00011.safete(…): 92%|#########2| 4.56G/4.95G [00:02<00:00, 2.52GB/s]
text_encoder/model-00004-of-00011.safete(…): 92%|#########2| 4.56G/4.95G [00:02<00:00, 2.52GB/s]
text_encoder/model-00004-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 2.42GB/s]
text_encoder/model-00004-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 2.42GB/s]
text_encoder/model-00004-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.79GB/s]
text_encoder/model-00004-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.79GB/s] |
|
|
|
text_encoder/model-00005-of-00011.safete(…): 0%| | 0.00/4.91G [00:00<?, ?B/s]
text_encoder/model-00005-of-00011.safete(…): 0%| | 0.00/4.91G [00:00<?, ?B/s]
text_encoder/model-00005-of-00011.safete(…): 0%| | 614k/4.91G [00:00<2:01:41, 672kB/s]
text_encoder/model-00005-of-00011.safete(…): 0%| | 614k/4.91G [00:00<2:01:41, 672kB/s]
text_encoder/model-00005-of-00011.safete(…): 14%|#3 | 671M/4.91G [00:01<00:07, 576MB/s]
text_encoder/model-00005-of-00011.safete(…): 14%|#3 | 671M/4.91G [00:01<00:07, 576MB/s]
text_encoder/model-00005-of-00011.safete(…): 44%|####3 | 2.15G/4.91G [00:01<00:01, 1.68GB/s]
text_encoder/model-00005-of-00011.safete(…): 44%|####3 | 2.15G/4.91G [00:01<00:01, 1.68GB/s]
text_encoder/model-00005-of-00011.safete(…): 53%|#####3 | 2.62G/4.91G [00:01<00:01, 1.88GB/s]
text_encoder/model-00005-of-00011.safete(…): 53%|#####3 | 2.62G/4.91G [00:01<00:01, 1.88GB/s]
text_encoder/model-00005-of-00011.safete(…): 67%|######7 | 3.29G/4.91G [00:02<00:00, 2.36GB/s]
text_encoder/model-00005-of-00011.safete(…): 67%|######7 | 3.29G/4.91G [00:02<00:00, 2.36GB/s]
text_encoder/model-00005-of-00011.safete(…): 78%|#######8 | 3.84G/4.91G [00:02<00:00, 2.81GB/s]
text_encoder/model-00005-of-00011.safete(…): 78%|#######8 | 3.84G/4.91G [00:02<00:00, 2.81GB/s]
text_encoder/model-00005-of-00011.safete(…): 86%|########6 | 4.24G/4.91G [00:02<00:00, 2.81GB/s]
text_encoder/model-00005-of-00011.safete(…): 86%|########6 | 4.24G/4.91G [00:02<00:00, 2.81GB/s]
text_encoder/model-00005-of-00011.safete(…): 96%|#########5| 4.71G/4.91G [00:02<00:00, 3.15GB/s]
text_encoder/model-00005-of-00011.safete(…): 96%|#########5| 4.71G/4.91G [00:02<00:00, 3.15GB/s]
text_encoder/model-00005-of-00011.safete(…): 100%|##########| 4.91G/4.91G [00:02<00:00, 1.92GB/s]
text_encoder/model-00005-of-00011.safete(…): 100%|##########| 4.91G/4.91G [00:02<00:00, 1.92GB/s] |
|
|
|
text_encoder/model-00006-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00006-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00006-of-00011.safete(…): 0%| | 34.7k/4.95G [00:00<38:02:44, 36.2kB/s]
text_encoder/model-00006-of-00011.safete(…): 0%| | 34.7k/4.95G [00:00<38:02:44, 36.2kB/s]
text_encoder/model-00006-of-00011.safete(…): 20%|## | 1.01G/4.95G [00:01<00:05, 757MB/s]
text_encoder/model-00006-of-00011.safete(…): 20%|## | 1.01G/4.95G [00:01<00:05, 757MB/s]
text_encoder/model-00006-of-00011.safete(…): 43%|####3 | 2.15G/4.95G [00:02<00:02, 1.38GB/s]
text_encoder/model-00006-of-00011.safete(…): 43%|####3 | 2.15G/4.95G [00:02<00:02, 1.38GB/s]
text_encoder/model-00006-of-00011.safete(…): 58%|#####8 | 2.89G/4.95G [00:02<00:01, 1.95GB/s]
text_encoder/model-00006-of-00011.safete(…): 58%|#####8 | 2.89G/4.95G [00:02<00:01, 1.95GB/s]
text_encoder/model-00006-of-00011.safete(…): 66%|######6 | 3.29G/4.95G [00:02<00:00, 1.99GB/s]
text_encoder/model-00006-of-00011.safete(…): 66%|######6 | 3.29G/4.95G [00:02<00:00, 1.99GB/s]
text_encoder/model-00006-of-00011.safete(…): 77%|#######7 | 3.83G/4.95G [00:02<00:00, 2.40GB/s]
text_encoder/model-00006-of-00011.safete(…): 77%|#######7 | 3.83G/4.95G [00:02<00:00, 2.40GB/s]
text_encoder/model-00006-of-00011.safete(…): 85%|########5 | 4.23G/4.95G [00:02<00:00, 2.61GB/s]
text_encoder/model-00006-of-00011.safete(…): 85%|########5 | 4.23G/4.95G [00:02<00:00, 2.61GB/s]
text_encoder/model-00006-of-00011.safete(…): 93%|#########3| 4.62G/4.95G [00:02<00:00, 2.67GB/s]
text_encoder/model-00006-of-00011.safete(…): 93%|#########3| 4.62G/4.95G [00:02<00:00, 2.67GB/s]
text_encoder/model-00006-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.78GB/s]
text_encoder/model-00006-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.78GB/s] |
|
|
|
text_encoder/model-00007-of-00011.safete(…): 0%| | 0.00/4.91G [00:00<?, ?B/s]
text_encoder/model-00007-of-00011.safete(…): 0%| | 0.00/4.91G [00:00<?, ?B/s]
text_encoder/model-00007-of-00011.safete(…): 0%| | 23.4k/4.91G [00:00<52:17:35, 26.1kB/s]
text_encoder/model-00007-of-00011.safete(…): 0%| | 23.4k/4.91G [00:00<52:17:35, 26.1kB/s]
text_encoder/model-00007-of-00011.safete(…): 19%|#9 | 941M/4.91G [00:01<00:04, 805MB/s]
text_encoder/model-00007-of-00011.safete(…): 19%|#9 | 941M/4.91G [00:01<00:04, 805MB/s]
text_encoder/model-00007-of-00011.safete(…): 42%|####2 | 2.08G/4.91G [00:01<00:02, 1.41GB/s]
text_encoder/model-00007-of-00011.safete(…): 42%|####2 | 2.08G/4.91G [00:01<00:02, 1.41GB/s]
text_encoder/model-00007-of-00011.safete(…): 59%|#####8 | 2.89G/4.91G [00:02<00:01, 1.94GB/s]
text_encoder/model-00007-of-00011.safete(…): 59%|#####8 | 2.89G/4.91G [00:02<00:01, 1.94GB/s]
text_encoder/model-00007-of-00011.safete(…): 74%|#######3 | 3.62G/4.91G [00:02<00:00, 2.44GB/s]
text_encoder/model-00007-of-00011.safete(…): 74%|#######3 | 3.62G/4.91G [00:02<00:00, 2.44GB/s]
text_encoder/model-00007-of-00011.safete(…): 84%|########3 | 4.10G/4.91G [00:02<00:00, 2.47GB/s]
text_encoder/model-00007-of-00011.safete(…): 84%|########3 | 4.10G/4.91G [00:02<00:00, 2.47GB/s]
text_encoder/model-00007-of-00011.safete(…): 96%|#########5| 4.71G/4.91G [00:02<00:00, 2.95GB/s]
text_encoder/model-00007-of-00011.safete(…): 96%|#########5| 4.71G/4.91G [00:02<00:00, 2.95GB/s]
text_encoder/model-00007-of-00011.safete(…): 100%|##########| 4.91G/4.91G [00:02<00:00, 1.87GB/s]
text_encoder/model-00007-of-00011.safete(…): 100%|##########| 4.91G/4.91G [00:02<00:00, 1.87GB/s] |
|
|
|
text_encoder/model-00008-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00008-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00008-of-00011.safete(…): 0%| | 538k/4.95G [00:00<2:18:45, 595kB/s]
text_encoder/model-00008-of-00011.safete(…): 0%| | 538k/4.95G [00:00<2:18:45, 595kB/s]
text_encoder/model-00008-of-00011.safete(…): 20%|## | 1.01G/4.95G [00:01<00:04, 829MB/s]
text_encoder/model-00008-of-00011.safete(…): 20%|## | 1.01G/4.95G [00:01<00:04, 829MB/s]
text_encoder/model-00008-of-00011.safete(…): 43%|####3 | 2.15G/4.95G [00:02<00:02, 1.29GB/s]
text_encoder/model-00008-of-00011.safete(…): 43%|####3 | 2.15G/4.95G [00:02<00:02, 1.29GB/s]
text_encoder/model-00008-of-00011.safete(…): 70%|####### | 3.49G/4.95G [00:02<00:00, 2.34GB/s]
text_encoder/model-00008-of-00011.safete(…): 70%|####### | 3.49G/4.95G [00:02<00:00, 2.34GB/s]
text_encoder/model-00008-of-00011.safete(…): 81%|########1 | 4.03G/4.95G [00:02<00:00, 2.59GB/s]
text_encoder/model-00008-of-00011.safete(…): 81%|########1 | 4.03G/4.95G [00:02<00:00, 2.59GB/s]
text_encoder/model-00008-of-00011.safete(…): 92%|#########1| 4.55G/4.95G [00:02<00:00, 2.50GB/s]
text_encoder/model-00008-of-00011.safete(…): 92%|#########1| 4.55G/4.95G [00:02<00:00, 2.50GB/s]
text_encoder/model-00008-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 2.67GB/s]
text_encoder/model-00008-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 2.67GB/s]
text_encoder/model-00008-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.86GB/s]
text_encoder/model-00008-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.86GB/s] |
|
|
|
text_encoder/model-00009-of-00011.safete(…): 0%| | 0.00/4.91G [00:00<?, ?B/s]
text_encoder/model-00009-of-00011.safete(…): 0%| | 0.00/4.91G [00:00<?, ?B/s]
text_encoder/model-00009-of-00011.safete(…): 0%| | 13.2k/4.91G [00:00<90:46:21, 15.0kB/s]
text_encoder/model-00009-of-00011.safete(…): 0%| | 13.2k/4.91G [00:00<90:46:21, 15.0kB/s]
text_encoder/model-00009-of-00011.safete(…): 21%|## | 1.01G/4.91G [00:01<00:05, 713MB/s]
text_encoder/model-00009-of-00011.safete(…): 21%|## | 1.01G/4.91G [00:01<00:05, 713MB/s]
text_encoder/model-00009-of-00011.safete(…): 49%|####8 | 2.40G/4.91G [00:01<00:01, 1.92GB/s]
text_encoder/model-00009-of-00011.safete(…): 49%|####8 | 2.40G/4.91G [00:01<00:01, 1.92GB/s]
text_encoder/model-00009-of-00011.safete(…): 63%|######2 | 3.07G/4.91G [00:10<00:08, 218MB/s]
text_encoder/model-00009-of-00011.safete(…): 63%|######2 | 3.07G/4.91G [00:10<00:08, 218MB/s]
text_encoder/model-00009-of-00011.safete(…): 81%|########1 | 4.00G/4.91G [00:10<00:02, 347MB/s]
text_encoder/model-00009-of-00011.safete(…): 81%|########1 | 4.00G/4.91G [00:10<00:02, 347MB/s]
text_encoder/model-00009-of-00011.safete(…): 95%|#########4| 4.64G/4.91G [00:11<00:00, 466MB/s]
text_encoder/model-00009-of-00011.safete(…): 95%|#########4| 4.64G/4.91G [00:11<00:00, 466MB/s]
text_encoder/model-00009-of-00011.safete(…): 100%|##########| 4.91G/4.91G [00:11<00:00, 443MB/s]
text_encoder/model-00009-of-00011.safete(…): 100%|##########| 4.91G/4.91G [00:11<00:00, 443MB/s] |
|
|
|
text_encoder/model-00010-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00010-of-00011.safete(…): 0%| | 0.00/4.95G [00:00<?, ?B/s]
text_encoder/model-00010-of-00011.safete(…): 0%| | 12.7k/4.95G [00:00<100:18:55, 13.7kB/s]
text_encoder/model-00010-of-00011.safete(…): 0%| | 12.7k/4.95G [00:00<100:18:55, 13.7kB/s]
text_encoder/model-00010-of-00011.safete(…): 8%|7 | 388M/4.95G [00:01<00:08, 516MB/s]
text_encoder/model-00010-of-00011.safete(…): 8%|7 | 388M/4.95G [00:01<00:08, 516MB/s]
text_encoder/model-00010-of-00011.safete(…): 17%|#7 | 845M/4.95G [00:01<00:07, 583MB/s]
text_encoder/model-00010-of-00011.safete(…): 17%|#7 | 845M/4.95G [00:01<00:07, 583MB/s]
text_encoder/model-00010-of-00011.safete(…): 25%|##5 | 1.26G/4.95G [00:01<00:03, 947MB/s]
text_encoder/model-00010-of-00011.safete(…): 25%|##5 | 1.26G/4.95G [00:01<00:03, 947MB/s]
text_encoder/model-00010-of-00011.safete(…): 35%|###4 | 1.72G/4.95G [00:01<00:02, 1.42GB/s]
text_encoder/model-00010-of-00011.safete(…): 35%|###4 | 1.72G/4.95G [00:01<00:02, 1.42GB/s]
text_encoder/model-00010-of-00011.safete(…): 45%|####4 | 2.23G/4.95G [00:02<00:01, 1.98GB/s]
text_encoder/model-00010-of-00011.safete(…): 45%|####4 | 2.23G/4.95G [00:02<00:01, 1.98GB/s]
text_encoder/model-00010-of-00011.safete(…): 54%|#####3 | 2.67G/4.95G [00:02<00:00, 2.41GB/s]
text_encoder/model-00010-of-00011.safete(…): 54%|#####3 | 2.67G/4.95G [00:02<00:00, 2.41GB/s]
text_encoder/model-00010-of-00011.safete(…): 64%|######3 | 3.16G/4.95G [00:02<00:00, 2.92GB/s]
text_encoder/model-00010-of-00011.safete(…): 64%|######3 | 3.16G/4.95G [00:02<00:00, 2.92GB/s]
text_encoder/model-00010-of-00011.safete(…): 73%|#######2 | 3.60G/4.95G [00:02<00:00, 3.14GB/s]
text_encoder/model-00010-of-00011.safete(…): 73%|#######2 | 3.60G/4.95G [00:02<00:00, 3.14GB/s]
text_encoder/model-00010-of-00011.safete(…): 84%|########4 | 4.18G/4.95G [00:02<00:00, 3.77GB/s]
text_encoder/model-00010-of-00011.safete(…): 84%|########4 | 4.18G/4.95G [00:02<00:00, 3.77GB/s]
text_encoder/model-00010-of-00011.safete(…): 94%|#########3| 4.65G/4.95G [00:02<00:00, 4.00GB/s]
text_encoder/model-00010-of-00011.safete(…): 94%|#########3| 4.65G/4.95G [00:02<00:00, 4.00GB/s]
text_encoder/model-00010-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.68GB/s]
text_encoder/model-00010-of-00011.safete(…): 100%|##########| 4.95G/4.95G [00:02<00:00, 1.68GB/s] |
|
|
|
text_encoder/model-00011-of-00011.safete(…): 0%| | 0.00/2.69G [00:00<?, ?B/s]
text_encoder/model-00011-of-00011.safete(…): 0%| | 0.00/2.69G [00:00<?, ?B/s]
text_encoder/model-00011-of-00011.safete(…): 0%| | 12.7k/2.69G [00:00<30:00:08, 24.9kB/s]
text_encoder/model-00011-of-00011.safete(…): 0%| | 12.7k/2.69G [00:00<30:00:08, 24.9kB/s]
text_encoder/model-00011-of-00011.safete(…): 11%|#1 | 304M/2.69G [00:00<00:03, 660MB/s]
text_encoder/model-00011-of-00011.safete(…): 11%|#1 | 304M/2.69G [00:00<00:03, 660MB/s]
text_encoder/model-00011-of-00011.safete(…): 25%|##4 | 665M/2.69G [00:00<00:01, 1.24GB/s]
text_encoder/model-00011-of-00011.safete(…): 25%|##4 | 665M/2.69G [00:00<00:01, 1.24GB/s]
text_encoder/model-00011-of-00011.safete(…): 33%|###2 | 878M/2.69G [00:02<00:06, 263MB/s]
text_encoder/model-00011-of-00011.safete(…): 33%|###2 | 878M/2.69G [00:02<00:06, 263MB/s]
text_encoder/model-00011-of-00011.safete(…): 47%|####7 | 1.27G/2.69G [00:02<00:02, 474MB/s]
text_encoder/model-00011-of-00011.safete(…): 47%|####7 | 1.27G/2.69G [00:02<00:02, 474MB/s]
text_encoder/model-00011-of-00011.safete(…): 66%|######6 | 1.78G/2.69G [00:02<00:01, 829MB/s]
text_encoder/model-00011-of-00011.safete(…): 66%|######6 | 1.78G/2.69G [00:02<00:01, 829MB/s]
text_encoder/model-00011-of-00011.safete(…): 87%|########6 | 2.33G/2.69G [00:03<00:00, 1.27GB/s]
text_encoder/model-00011-of-00011.safete(…): 87%|########6 | 2.33G/2.69G [00:03<00:00, 1.27GB/s]
text_encoder/model-00011-of-00011.safete(…): 100%|##########| 2.69G/2.69G [00:03<00:00, 850MB/s]
text_encoder/model-00011-of-00011.safete(…): 100%|##########| 2.69G/2.69G [00:03<00:00, 850MB/s] |
|
|
|
Loading checkpoint shards: 0%| | 0/11 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/11 [00:00<?, ?it/s]
Loading checkpoint shards: 9%|9 | 1/11 [00:05<00:50, 5.10s/it]
Loading checkpoint shards: 9%|9 | 1/11 [00:05<00:50, 5.10s/it]
Loading checkpoint shards: 18%|#8 | 2/11 [00:06<00:27, 3.06s/it]
Loading checkpoint shards: 18%|#8 | 2/11 [00:06<00:27, 3.06s/it]
Loading checkpoint shards: 27%|##7 | 3/11 [00:09<00:22, 2.79s/it]
Loading checkpoint shards: 27%|##7 | 3/11 [00:09<00:22, 2.79s/it]
Loading checkpoint shards: 36%|###6 | 4/11 [00:13<00:23, 3.39s/it]
Loading checkpoint shards: 36%|###6 | 4/11 [00:13<00:23, 3.39s/it]
Loading checkpoint shards: 45%|####5 | 5/11 [00:17<00:21, 3.54s/it]
Loading checkpoint shards: 45%|####5 | 5/11 [00:17<00:21, 3.54s/it]
Loading checkpoint shards: 55%|#####4 | 6/11 [00:20<00:17, 3.58s/it]
Loading checkpoint shards: 55%|#####4 | 6/11 [00:20<00:17, 3.58s/it]
Loading checkpoint shards: 64%|######3 | 7/11 [00:24<00:14, 3.51s/it]
Loading checkpoint shards: 64%|######3 | 7/11 [00:24<00:14, 3.51s/it]
Loading checkpoint shards: 73%|#######2 | 8/11 [00:27<00:10, 3.52s/it]
Loading checkpoint shards: 73%|#######2 | 8/11 [00:27<00:10, 3.52s/it]
Loading checkpoint shards: 82%|########1 | 9/11 [00:32<00:07, 3.79s/it]
Loading checkpoint shards: 82%|########1 | 9/11 [00:32<00:07, 3.79s/it]
Loading checkpoint shards: 91%|######### | 10/11 [00:36<00:03, 3.95s/it]
Loading checkpoint shards: 91%|######### | 10/11 [00:36<00:03, 3.95s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:39<00:00, 3.60s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:39<00:00, 3.60s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:39<00:00, 3.58s/it]
Loading checkpoint shards: 100%|##########| 11/11 [00:39<00:00, 3.58s/it] |
|
|
|
generation_config.json: 0%| | 0.00/168 [00:00<?, ?B/s]
generation_config.json: 0%| | 0.00/168 [00:00<?, ?B/s]
generation_config.json: 100%|##########| 168/168 [00:00<00:00, 1.78MB/s]
generation_config.json: 100%|##########| 168/168 [00:00<00:00, 1.78MB/s] |
|
|
| Quantizing Text Encoder |
| Loading VAEs and other components |
|
config.json: 0.00B [00:00, ?B/s]
config.json: 0.00B [00:00, ?B/s]
config.json: 1.32kB [00:00, 746kB/s]
config.json: 1.32kB [00:00, 746kB/s] |
|
|
|
vae/diffusion_pytorch_model.safetensors: 0%| | 0.00/2.44G [00:00<?, ?B/s]
vae/diffusion_pytorch_model.safetensors: 0%| | 0.00/2.44G [00:00<?, ?B/s]
vae/diffusion_pytorch_model.safetensors: 0%| | 1.20M/2.44G [00:00<24:46, 1.64MB/s]
vae/diffusion_pytorch_model.safetensors: 0%| | 1.20M/2.44G [00:00<24:46, 1.64MB/s]
vae/diffusion_pytorch_model.safetensors: 8%|7 | 191M/2.44G [00:01<00:09, 233MB/s]
vae/diffusion_pytorch_model.safetensors: 8%|7 | 191M/2.44G [00:01<00:09, 233MB/s]
vae/diffusion_pytorch_model.safetensors: 44%|####4 | 1.08G/2.44G [00:01<00:00, 1.45GB/s]
vae/diffusion_pytorch_model.safetensors: 44%|####4 | 1.08G/2.44G [00:01<00:00, 1.45GB/s]
vae/diffusion_pytorch_model.safetensors: 64%|######4 | 1.57G/2.44G [00:01<00:00, 2.01GB/s]
vae/diffusion_pytorch_model.safetensors: 64%|######4 | 1.57G/2.44G [00:01<00:00, 2.01GB/s]
vae/diffusion_pytorch_model.safetensors: 81%|######## | 1.98G/2.44G [00:01<00:00, 2.28GB/s]
vae/diffusion_pytorch_model.safetensors: 81%|######## | 1.98G/2.44G [00:01<00:00, 2.28GB/s]
vae/diffusion_pytorch_model.safetensors: 97%|#########7| 2.38G/2.44G [00:01<00:00, 2.49GB/s]
vae/diffusion_pytorch_model.safetensors: 97%|#########7| 2.38G/2.44G [00:01<00:00, 2.49GB/s]
vae/diffusion_pytorch_model.safetensors: 100%|##########| 2.44G/2.44G [00:01<00:00, 1.57GB/s]
vae/diffusion_pytorch_model.safetensors: 100%|##########| 2.44G/2.44G [00:01<00:00, 1.57GB/s] |
|
|
|
config.json: 0%| | 0.00/505 [00:00<?, ?B/s]
config.json: 0%| | 0.00/505 [00:00<?, ?B/s]
config.json: 100%|##########| 505/505 [00:00<00:00, 2.91MB/s]
config.json: 100%|##########| 505/505 [00:00<00:00, 2.91MB/s] |
|
|
|
audio_vae/diffusion_pytorch_model.safete(…): 0%| | 0.00/107M [00:00<?, ?B/s]
audio_vae/diffusion_pytorch_model.safete(…): 0%| | 0.00/107M [00:00<?, ?B/s]
audio_vae/diffusion_pytorch_model.safete(…): 0%| | 242k/107M [00:00<02:52, 617kB/s]
audio_vae/diffusion_pytorch_model.safete(…): 0%| | 242k/107M [00:00<02:52, 617kB/s]
audio_vae/diffusion_pytorch_model.safete(…): 60%|#####9 | 63.8M/107M [00:00<00:00, 143MB/s]
audio_vae/diffusion_pytorch_model.safete(…): 60%|#####9 | 63.8M/107M [00:00<00:00, 143MB/s]
audio_vae/diffusion_pytorch_model.safete(…): 100%|##########| 107M/107M [00:00<00:00, 161MB/s]
audio_vae/diffusion_pytorch_model.safete(…): 100%|##########| 107M/107M [00:00<00:00, 161MB/s] |
|
|
|
config.json: 0%| | 0.00/649 [00:00<?, ?B/s]
config.json: 0%| | 0.00/649 [00:00<?, ?B/s]
config.json: 100%|##########| 649/649 [00:00<00:00, 3.06MB/s]
config.json: 100%|##########| 649/649 [00:00<00:00, 3.06MB/s] |
|
|
|
connectors/diffusion_pytorch_model.safet(…): 0%| | 0.00/2.86G [00:00<?, ?B/s]
connectors/diffusion_pytorch_model.safet(…): 0%| | 0.00/2.86G [00:00<?, ?B/s]
connectors/diffusion_pytorch_model.safet(…): 0%| | 2.86M/2.86G [00:00<14:31, 3.28MB/s]
connectors/diffusion_pytorch_model.safet(…): 0%| | 2.86M/2.86G [00:00<14:31, 3.28MB/s]
connectors/diffusion_pytorch_model.safet(…): 23%|##3 | 671M/2.86G [00:01<00:03, 716MB/s]
connectors/diffusion_pytorch_model.safet(…): 23%|##3 | 671M/2.86G [00:01<00:03, 716MB/s]
connectors/diffusion_pytorch_model.safet(…): 45%|####4 | 1.28G/2.86G [00:01<00:01, 1.38GB/s]
connectors/diffusion_pytorch_model.safet(…): 45%|####4 | 1.28G/2.86G [00:01<00:01, 1.38GB/s]
connectors/diffusion_pytorch_model.safet(…): 71%|#######1 | 2.05G/2.86G [00:01<00:00, 2.30GB/s]
connectors/diffusion_pytorch_model.safet(…): 71%|#######1 | 2.05G/2.86G [00:01<00:00, 2.30GB/s]
connectors/diffusion_pytorch_model.safet(…): 88%|########8 | 2.53G/2.86G [00:01<00:00, 2.13GB/s]
connectors/diffusion_pytorch_model.safet(…): 88%|########8 | 2.53G/2.86G [00:01<00:00, 2.13GB/s]
connectors/diffusion_pytorch_model.safet(…): 100%|##########| 2.86G/2.86G [00:01<00:00, 1.54GB/s]
connectors/diffusion_pytorch_model.safet(…): 100%|##########| 2.86G/2.86G [00:01<00:00, 1.54GB/s] |
|
|
|
config.json: 0%| | 0.00/544 [00:00<?, ?B/s]
config.json: 0%| | 0.00/544 [00:00<?, ?B/s]
config.json: 100%|##########| 544/544 [00:00<00:00, 1.54MB/s]
config.json: 100%|##########| 544/544 [00:00<00:00, 1.54MB/s] |
|
|
|
vocoder/diffusion_pytorch_model.safetens(…): 0%| | 0.00/111M [00:00<?, ?B/s]
vocoder/diffusion_pytorch_model.safetens(…): 0%| | 0.00/111M [00:00<?, ?B/s]
vocoder/diffusion_pytorch_model.safetens(…): 13%|#3 | 14.5M/111M [00:00<00:02, 32.4MB/s]
vocoder/diffusion_pytorch_model.safetens(…): 13%|#3 | 14.5M/111M [00:00<00:02, 32.4MB/s]
vocoder/diffusion_pytorch_model.safetens(…): 58%|#####8 | 64.8M/111M [00:00<00:00, 112MB/s]
vocoder/diffusion_pytorch_model.safetens(…): 58%|#####8 | 64.8M/111M [00:00<00:00, 112MB/s]
vocoder/diffusion_pytorch_model.safetens(…): 100%|##########| 111M/111M [00:00<00:00, 154MB/s]
vocoder/diffusion_pytorch_model.safetens(…): 100%|##########| 111M/111M [00:00<00:00, 154MB/s] |
|
|
| Making pipe |
| Preparing Model |
| Model Loaded |
| create LoRA network. base dim (rank): 32, alpha: 32 |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None |
| apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16 |
| create LoRA for Text Encoder: 0 modules. |
| create LoRA for U-Net: 1344 modules. |
| enable LoRA for U-Net |
| Dataset: /app/ai-toolkit/datasets/49 |
| - Preprocessing video dimensions |
|
0%| | 0/140 [00:00<?, ?it/s]
0%| | 0/140 [00:00<?, ?it/s]
8%|7 | 11/140 [00:00<00:01, 102.69it/s]
8%|7 | 11/140 [00:00<00:01, 102.69it/s]
19%|#8 | 26/140 [00:00<00:00, 126.82it/s]
19%|#8 | 26/140 [00:00<00:00, 126.82it/s]
31%|### | 43/140 [00:00<00:00, 143.80it/s]
31%|### | 43/140 [00:00<00:00, 143.80it/s]
41%|####1 | 58/140 [00:00<00:00, 143.46it/s]
41%|####1 | 58/140 [00:00<00:00, 143.46it/s]
100%|##########| 140/140 [00:00<00:00, 283.02it/s]
100%|##########| 140/140 [00:00<00:00, 283.02it/s] |
|
|
| - Found 140 videos |
| Bucket sizes for /app/ai-toolkit/datasets/49: |
| 576x416: 78 files |
| 576x448: 44 files |
| 544x416: 16 files |
| 512x384: 2 files |
| 4 buckets made |
| Caching latents for /app/ai-toolkit/datasets/49 |
| - Saving latents to disk |
|
Caching latents to disk: 0%| | 0/140 [00:00<?, ?it/s]
Caching latents to disk: 0%| | 0/140 [00:00<?, ?it/s]
Caching latents to disk: 1%| | 1/140 [00:05<12:28, 5.39s/it]
Caching latents to disk: 1%| | 1/140 [00:05<12:28, 5.39s/it]
Caching latents to disk: 1%|1 | 2/140 [00:10<12:04, 5.25s/it]
Caching latents to disk: 1%|1 | 2/140 [00:10<12:04, 5.25s/it]
Caching latents to disk: 2%|2 | 3/140 [00:15<11:36, 5.08s/it]
Caching latents to disk: 2%|2 | 3/140 [00:15<11:36, 5.08s/it]
Caching latents to disk: 3%|2 | 4/140 [00:20<11:05, 4.89s/it]
Caching latents to disk: 3%|2 | 4/140 [00:20<11:05, 4.89s/it]
Caching latents to disk: 4%|3 | 5/140 [00:24<10:48, 4.81s/it]
Caching latents to disk: 4%|3 | 5/140 [00:24<10:48, 4.81s/it]
Caching latents to disk: 4%|4 | 6/140 [00:29<10:48, 4.84s/it]
Caching latents to disk: 4%|4 | 6/140 [00:29<10:48, 4.84s/it]
Caching latents to disk: 5%|5 | 7/140 [00:34<10:30, 4.74s/it]
Caching latents to disk: 5%|5 | 7/140 [00:34<10:30, 4.74s/it]
Caching latents to disk: 6%|5 | 8/140 [00:39<10:40, 4.85s/it]
Caching latents to disk: 6%|5 | 8/140 [00:39<10:40, 4.85s/it]
Caching latents to disk: 6%|6 | 9/140 [00:43<10:32, 4.83s/it]
Caching latents to disk: 6%|6 | 9/140 [00:43<10:32, 4.83s/it]
Caching latents to disk: 7%|7 | 10/140 [00:49<10:38, 4.91s/it]
Caching latents to disk: 7%|7 | 10/140 [00:49<10:38, 4.91s/it]
Caching latents to disk: 8%|7 | 11/140 [00:53<10:17, 4.79s/it]
Caching latents to disk: 8%|7 | 11/140 [00:53<10:17, 4.79s/it]
Caching latents to disk: 9%|8 | 12/140 [00:57<09:26, 4.43s/it]
Caching latents to disk: 9%|8 | 12/140 [00:57<09:26, 4.43s/it]
Caching latents to disk: 9%|9 | 13/140 [01:01<09:32, 4.51s/it]
Caching latents to disk: 9%|9 | 13/140 [01:01<09:32, 4.51s/it]
Caching latents to disk: 10%|# | 14/140 [01:07<10:05, 4.80s/it]
Caching latents to disk: 10%|# | 14/140 [01:07<10:05, 4.80s/it]
Caching latents to disk: 11%|# | 15/140 [01:12<10:20, 4.96s/it]
Caching latents to disk: 11%|# | 15/140 [01:12<10:20, 4.96s/it]
Caching latents to disk: 11%|#1 | 16/140 [01:15<09:01, 4.36s/it]
Caching latents to disk: 11%|#1 | 16/140 [01:15<09:01, 4.36s/it]
Caching latents to disk: 12%|#2 | 17/140 [01:20<09:15, 4.51s/it]
Caching latents to disk: 12%|#2 | 17/140 [01:20<09:15, 4.51s/it]
Caching latents to disk: 13%|#2 | 18/140 [01:25<09:19, 4.59s/it]
Caching latents to disk: 13%|#2 | 18/140 [01:25<09:19, 4.59s/it]
Caching latents to disk: 14%|#3 | 19/140 [01:27<07:58, 3.95s/it]
Caching latents to disk: 14%|#3 | 19/140 [01:27<07:58, 3.95s/it]
Caching latents to disk: 14%|#4 | 20/140 [01:30<07:20, 3.67s/it]
Caching latents to disk: 14%|#4 | 20/140 [01:30<07:20, 3.67s/it]
Caching latents to disk: 15%|#5 | 21/140 [01:33<06:57, 3.51s/it]
Caching latents to disk: 15%|#5 | 21/140 [01:33<06:57, 3.51s/it]
Caching latents to disk: 16%|#5 | 22/140 [01:36<06:30, 3.31s/it]
Caching latents to disk: 16%|#5 | 22/140 [01:36<06:30, 3.31s/it]
Caching latents to disk: 16%|#6 | 23/140 [01:42<07:45, 3.98s/it]
Caching latents to disk: 16%|#6 | 23/140 [01:42<07:45, 3.98s/it]
Caching latents to disk: 17%|#7 | 24/140 [01:47<08:13, 4.25s/it]
Caching latents to disk: 17%|#7 | 24/140 [01:47<08:13, 4.25s/it]
Caching latents to disk: 18%|#7 | 25/140 [01:51<08:22, 4.37s/it]
Caching latents to disk: 18%|#7 | 25/140 [01:51<08:22, 4.37s/it]
Caching latents to disk: 19%|#8 | 26/140 [01:57<08:48, 4.63s/it]
Caching latents to disk: 19%|#8 | 26/140 [01:57<08:48, 4.63s/it]
Caching latents to disk: 19%|#9 | 27/140 [01:59<07:40, 4.08s/it]
Caching latents to disk: 19%|#9 | 27/140 [01:59<07:40, 4.08s/it]
Caching latents to disk: 20%|## | 28/140 [02:04<08:05, 4.34s/it]
Caching latents to disk: 20%|## | 28/140 [02:04<08:05, 4.34s/it]
Caching latents to disk: 21%|## | 29/140 [02:09<08:22, 4.53s/it]
Caching latents to disk: 21%|## | 29/140 [02:09<08:22, 4.53s/it]
Caching latents to disk: 21%|##1 | 30/140 [02:14<08:22, 4.56s/it]
Caching latents to disk: 21%|##1 | 30/140 [02:14<08:22, 4.56s/it]
Caching latents to disk: 22%|##2 | 31/140 [02:19<08:27, 4.65s/it]
Caching latents to disk: 22%|##2 | 31/140 [02:19<08:27, 4.65s/it]
Caching latents to disk: 23%|##2 | 32/140 [02:23<08:01, 4.46s/it]
Caching latents to disk: 23%|##2 | 32/140 [02:23<08:01, 4.46s/it]
Caching latents to disk: 24%|##3 | 33/140 [02:27<07:56, 4.45s/it]
Caching latents to disk: 24%|##3 | 33/140 [02:27<07:56, 4.45s/it]
Caching latents to disk: 24%|##4 | 34/140 [02:32<08:08, 4.61s/it]
Caching latents to disk: 24%|##4 | 34/140 [02:32<08:08, 4.61s/it]
Caching latents to disk: 25%|##5 | 35/140 [02:37<08:07, 4.65s/it]
Caching latents to disk: 25%|##5 | 35/140 [02:37<08:07, 4.65s/it]
Caching latents to disk: 26%|##5 | 36/140 [02:42<08:05, 4.67s/it]
Caching latents to disk: 26%|##5 | 36/140 [02:42<08:05, 4.67s/it]
Caching latents to disk: 26%|##6 | 37/140 [02:46<07:58, 4.65s/it]
Caching latents to disk: 26%|##6 | 37/140 [02:46<07:58, 4.65s/it]
Caching latents to disk: 27%|##7 | 38/140 [02:51<08:01, 4.72s/it]
Caching latents to disk: 27%|##7 | 38/140 [02:51<08:01, 4.72s/it]
Caching latents to disk: 28%|##7 | 39/140 [02:55<07:30, 4.46s/it]
Caching latents to disk: 28%|##7 | 39/140 [02:55<07:30, 4.46s/it]
Caching latents to disk: 29%|##8 | 40/140 [02:58<06:38, 3.98s/it]
Caching latents to disk: 29%|##8 | 40/140 [02:58<06:38, 3.98s/it]
Caching latents to disk: 29%|##9 | 41/140 [03:02<06:51, 4.16s/it]
Caching latents to disk: 29%|##9 | 41/140 [03:02<06:51, 4.16s/it]
Caching latents to disk: 30%|### | 42/140 [03:07<07:01, 4.30s/it]
Caching latents to disk: 30%|### | 42/140 [03:07<07:01, 4.30s/it]
Caching latents to disk: 31%|### | 43/140 [03:10<06:14, 3.86s/it]
Caching latents to disk: 31%|### | 43/140 [03:10<06:14, 3.86s/it]
Caching latents to disk: 31%|###1 | 44/140 [03:15<06:35, 4.12s/it]
Caching latents to disk: 31%|###1 | 44/140 [03:15<06:35, 4.12s/it]
Caching latents to disk: 32%|###2 | 45/140 [03:18<06:04, 3.84s/it]
Caching latents to disk: 32%|###2 | 45/140 [03:18<06:04, 3.84s/it]
Caching latents to disk: 33%|###2 | 46/140 [03:22<06:03, 3.86s/it]
Caching latents to disk: 33%|###2 | 46/140 [03:22<06:03, 3.86s/it]
Caching latents to disk: 34%|###3 | 47/140 [03:27<06:33, 4.24s/it]
Caching latents to disk: 34%|###3 | 47/140 [03:27<06:33, 4.24s/it]
Caching latents to disk: 34%|###4 | 48/140 [03:32<06:58, 4.55s/it]
Caching latents to disk: 34%|###4 | 48/140 [03:32<06:58, 4.55s/it]
Caching latents to disk: 35%|###5 | 49/140 [03:37<07:08, 4.71s/it]
Caching latents to disk: 35%|###5 | 49/140 [03:37<07:08, 4.71s/it]
Caching latents to disk: 36%|###5 | 50/140 [03:42<07:01, 4.68s/it]
Caching latents to disk: 36%|###5 | 50/140 [03:42<07:01, 4.68s/it]
Caching latents to disk: 36%|###6 | 51/140 [03:46<06:52, 4.64s/it]
Caching latents to disk: 36%|###6 | 51/140 [03:46<06:52, 4.64s/it]
Caching latents to disk: 37%|###7 | 52/140 [03:51<07:00, 4.77s/it]
Caching latents to disk: 37%|###7 | 52/140 [03:51<07:00, 4.77s/it]
Caching latents to disk: 38%|###7 | 53/140 [03:56<06:57, 4.80s/it]
Caching latents to disk: 38%|###7 | 53/140 [03:56<06:57, 4.80s/it]
Caching latents to disk: 39%|###8 | 54/140 [04:01<07:00, 4.90s/it]
Caching latents to disk: 39%|###8 | 54/140 [04:01<07:00, 4.90s/it]
Caching latents to disk: 39%|###9 | 55/140 [04:06<07:00, 4.95s/it]
Caching latents to disk: 39%|###9 | 55/140 [04:06<07:00, 4.95s/it]
Caching latents to disk: 40%|#### | 56/140 [04:11<06:36, 4.73s/it]
Caching latents to disk: 40%|#### | 56/140 [04:11<06:36, 4.73s/it]
Caching latents to disk: 41%|#### | 57/140 [04:17<06:59, 5.05s/it]
Caching latents to disk: 41%|#### | 57/140 [04:17<06:59, 5.05s/it]
Caching latents to disk: 41%|####1 | 58/140 [04:21<06:50, 5.01s/it]
Caching latents to disk: 41%|####1 | 58/140 [04:21<06:50, 5.01s/it]
Caching latents to disk: 42%|####2 | 59/140 [04:26<06:44, 5.00s/it]
Caching latents to disk: 42%|####2 | 59/140 [04:26<06:44, 5.00s/it]
Caching latents to disk: 43%|####2 | 60/140 [04:32<06:49, 5.12s/it]
Caching latents to disk: 43%|####2 | 60/140 [04:32<06:49, 5.12s/it]
Caching latents to disk: 44%|####3 | 61/140 [04:36<06:25, 4.88s/it]
Caching latents to disk: 44%|####3 | 61/140 [04:36<06:25, 4.88s/it]
Caching latents to disk: 44%|####4 | 62/140 [04:42<06:39, 5.12s/it]
Caching latents to disk: 44%|####4 | 62/140 [04:42<06:39, 5.12s/it]
Caching latents to disk: 45%|####5 | 63/140 [04:47<06:33, 5.11s/it]
Caching latents to disk: 45%|####5 | 63/140 [04:47<06:33, 5.11s/it]
Caching latents to disk: 46%|####5 | 64/140 [04:52<06:28, 5.11s/it]
Caching latents to disk: 46%|####5 | 64/140 [04:52<06:28, 5.11s/it]
Caching latents to disk: 46%|####6 | 65/140 [04:57<06:23, 5.11s/it]
Caching latents to disk: 46%|####6 | 65/140 [04:57<06:23, 5.11s/it]
Caching latents to disk: 47%|####7 | 66/140 [05:02<06:09, 5.00s/it]
Caching latents to disk: 47%|####7 | 66/140 [05:02<06:09, 5.00s/it]
Caching latents to disk: 48%|####7 | 67/140 [05:04<05:11, 4.26s/it]
Caching latents to disk: 48%|####7 | 67/140 [05:04<05:11, 4.26s/it]
Caching latents to disk: 49%|####8 | 68/140 [05:08<05:03, 4.22s/it]
Caching latents to disk: 49%|####8 | 68/140 [05:08<05:03, 4.22s/it]
Caching latents to disk: 49%|####9 | 69/140 [05:12<04:35, 3.87s/it]
Caching latents to disk: 49%|####9 | 69/140 [05:12<04:35, 3.87s/it]
Caching latents to disk: 50%|##### | 70/140 [05:16<04:33, 3.91s/it]
Caching latents to disk: 50%|##### | 70/140 [05:16<04:33, 3.91s/it]
Caching latents to disk: 100%|##########| 140/140 [05:16<00:00, 2.26s/it]
Caching latents to disk: 100%|##########| 140/140 [05:16<00:00, 2.26s/it] |
|
|
| Caching text_embeddings for /app/ai-toolkit/datasets/49 |
| - Saving text embeddings to disk |
|
Caching text embeddings to disk: 0%| | 0/140 [00:00<?, ?it/s]
Caching text embeddings to disk: 0%| | 0/140 [00:00<?, ?it/s] |
