upload mastur_copy checkpoints (ignore sqlite wal/shm)

mastur_copy/.job_config.json

@@ -0,0 +1,129 @@
+{
+  "job": "extension",
+  "config": {
+    "name": "mastur_copy",
+    "process": [
+      {
+        "type": "diffusion_trainer",
+        "training_folder": "/workspace/ai-toolkit/output",
+        "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+        "device": "cuda",
+        "trigger_word": null,
+        "performance_log_every": 10,
+        "network": {
+          "type": "lora",
+          "linear": 64,
+          "linear_alpha": 64,
+          "conv": 16,
+          "conv_alpha": 16,
+          "lokr_full_rank": true,
+          "lokr_factor": -1,
+          "network_kwargs": {
+            "ignore_if_contains": []
+          }
+        },
+        "save": {
+          "dtype": "bf16",
+          "save_every": 1000,
+          "max_step_saves_to_keep": 4,
+          "save_format": "diffusers",
+          "push_to_hub": false
+        },
+        "datasets": [
+          {
+            "folder_path": "/workspace/ai-toolkit/datasets/qwewq",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "woman masturbating pussy",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+              512
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 49,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+          }
+        ],
+        "train": {
+          "batch_size": 1,
+          "bypass_guidance_embedding": false,
+          "steps": 3000,
+          "gradient_accumulation": 1,
+          "train_unet": true,
+          "train_text_encoder": false,
+          "gradient_checkpointing": true,
+          "noise_scheduler": "flowmatch",
+          "optimizer": "adamw8bit",
+          "timestep_type": "linear",
+          "content_or_style": "balanced",
+          "optimizer_params": {
+            "weight_decay": 0.0001
+          },
+          "unload_text_encoder": false,
+          "cache_text_embeddings": false,
+          "lr": 0.0001,
+          "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+          },
+          "skip_first_sample": false,
+          "force_first_sample": false,
+          "disable_sampling": false,
+          "dtype": "bf16",
+          "diff_output_preservation": false,
+          "diff_output_preservation_multiplier": 1,
+          "diff_output_preservation_class": "person",
+          "switch_boundary_every": 1,
+          "loss_type": "mse",
+          "do_differential_guidance": true,
+          "differential_guidance_scale": 3
+        },
+        "logging": {
+          "log_every": 1,
+          "use_ui_logger": true
+        },
+        "model": {
+          "name_or_path": "ai-toolkit/Wan2.2-T2V-A14B-Diffusers-bf16",
+          "quantize": true,
+          "qtype": "qfloat8",
+          "quantize_te": true,
+          "qtype_te": "qfloat8",
+          "arch": "wan22_14b:t2v",
+          "low_vram": false,
+          "model_kwargs": {
+            "train_high_noise": true,
+            "train_low_noise": true
+          },
+          "layer_offloading": false,
+          "layer_offloading_text_encoder_percent": 1,
+          "layer_offloading_transformer_percent": 1
+        },
+        "sample": {
+          "sampler": "flowmatch",
+          "sample_every": 250,
+          "width": 1024,
+          "height": 1024,
+          "samples": [],
+          "neg": "",
+          "seed": 42,
+          "walk_seed": true,
+          "guidance_scale": 4,
+          "sample_steps": 25,
+          "num_frames": 41,
+          "fps": 16
+        }
+      }
+    ]
+  },
+  "meta": {
+    "name": "[name]",
+    "version": "1.0"
+  }
+}

mastur_copy/config.yaml

@@ -0,0 +1,108 @@
+job: extension
+config:
+  name: mastur_copy
+  process:
+  - type: diffusion_trainer
+    training_folder: /workspace/ai-toolkit/output
+    sqlite_db_path: /workspace/ai-toolkit/aitk_db.db
+    device: cuda
+    trigger_word: null
+    performance_log_every: 10
+    network:
+      type: lora
+      linear: 64
+      linear_alpha: 64
+      conv: 16
+      conv_alpha: 16
+      lokr_full_rank: true
+      lokr_factor: -1
+      network_kwargs:
+        ignore_if_contains: []
+    save:
+      dtype: bf16
+      save_every: 1000
+      max_step_saves_to_keep: 4
+      save_format: diffusers
+      push_to_hub: false
+    datasets:
+    - folder_path: /workspace/ai-toolkit/datasets/qwewq
+      mask_path: null
+      mask_min_value: 0.1
+      default_caption: woman masturbating pussy
+      caption_ext: txt
+      caption_dropout_rate: 0.05
+      cache_latents_to_disk: false
+      is_reg: false
+      network_weight: 1
+      resolution:
+      - 512
+      controls: []
+      shrink_video_to_frames: true
+      num_frames: 49
+      flip_x: false
+      flip_y: false
+      num_repeats: 1
+    train:
+      batch_size: 1
+      bypass_guidance_embedding: false
+      steps: 3000
+      gradient_accumulation: 1
+      train_unet: true
+      train_text_encoder: false
+      gradient_checkpointing: true
+      noise_scheduler: flowmatch
+      optimizer: adamw8bit
+      timestep_type: linear
+      content_or_style: balanced
+      optimizer_params:
+        weight_decay: 0.0001
+      unload_text_encoder: false
+      cache_text_embeddings: false
+      lr: 0.0001
+      ema_config:
+        use_ema: false
+        ema_decay: 0.99
+      skip_first_sample: false
+      force_first_sample: false
+      disable_sampling: false
+      dtype: bf16
+      diff_output_preservation: false
+      diff_output_preservation_multiplier: 1
+      diff_output_preservation_class: person
+      switch_boundary_every: 1
+      loss_type: mse
+      do_differential_guidance: true
+      differential_guidance_scale: 3
+    logging:
+      log_every: 1
+      use_ui_logger: true
+    model:
+      name_or_path: ai-toolkit/Wan2.2-T2V-A14B-Diffusers-bf16
+      quantize: true
+      qtype: qfloat8
+      quantize_te: true
+      qtype_te: qfloat8
+      arch: wan22_14b:t2v
+      low_vram: false
+      model_kwargs:
+        train_high_noise: true
+        train_low_noise: true
+      layer_offloading: false
+      layer_offloading_text_encoder_percent: 1
+      layer_offloading_transformer_percent: 1
+    sample:
+      sampler: flowmatch
+      sample_every: 250
+      width: 1024
+      height: 1024
+      samples: []
+      neg: ''
+      seed: 42
+      walk_seed: true
+      guidance_scale: 4
+      sample_steps: 25
+      num_frames: 41
+      fps: 16
+meta:
+  name: mastur_copy
+  version: '1.0'

mastur_copy/log.txt

@@ -0,0 +1,178 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/workspace/ai-toolkit/output",
+    "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": null,
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 64,
+        "linear_alpha": 64,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 1000,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/workspace/ai-toolkit/datasets/qwewq",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "woman masturbating pussy",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 49,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "linear",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse",
+        "do_differential_guidance": true,
+        "differential_guidance_scale": 3
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "ai-toolkit/Wan2.2-T2V-A14B-Diffusers-bf16",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "wan22_14b:t2v",
+        "low_vram": false,
+        "model_kwargs": {
+            "train_high_noise": true,
+            "train_low_noise": true
+        },
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 25,
+        "num_frames": 41,
+        "fps": 16
+    }
+}
+Changing sample extention to animated webp
+Using SQLite database at /workspace/ai-toolkit/aitk_db.db
+Job ID: "83e8ed4c-6743-434a-9d8f-364c179ac62e"
+
+#############################################
+# Running job: mastur_copy
+#############################################
+
+
+Running  1 process
+Loading Wan model
+Loading transformer 1
+
+
+Quantizing Transformer 1
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
  2%|2         | 1/40 [00:00<00:11,  3.32it/s]
  2%|2         | 1/40 [00:00<00:11,  3.32it/s]
  5%|5         | 2/40 [00:00<00:10,  3.79it/s]
  5%|5         | 2/40 [00:00<00:10,  3.79it/s]
  8%|7         | 3/40 [00:00<00:09,  3.92it/s]
  8%|7         | 3/40 [00:00<00:09,  3.92it/s]
 10%|#         | 4/40 [00:01<00:08,  4.06it/s]
 10%|#         | 4/40 [00:01<00:08,  4.06it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.13it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.13it/s]
 15%|#5        | 6/40 [00:01<00:08,  4.15it/s]
 15%|#5        | 6/40 [00:01<00:08,  4.15it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.22it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.22it/s]
 20%|##        | 8/40 [00:01<00:07,  4.26it/s]
 20%|##        | 8/40 [00:01<00:07,  4.26it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.30it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.30it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.32it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.32it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.25it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.25it/s]
 30%|###       | 12/40 [00:02<00:06,  4.29it/s]
 30%|###       | 12/40 [00:02<00:06,  4.29it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.32it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.32it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.35it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.35it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.37it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.37it/s]
 40%|####      | 16/40 [00:03<00:05,  4.38it/s]
 40%|####      | 16/40 [00:03<00:05,  4.38it/s]
 42%|####2     | 17/40 [00:04<00:05,  4.38it/s]
 42%|####2     | 17/40 [00:04<00:05,  4.38it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.39it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.39it/s]
 48%|####7     | 19/40 [00:04<00:04,  4.39it/s]
 48%|####7     | 19/40 [00:04<00:04,  4.39it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.38it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.38it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.38it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.38it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.39it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.39it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.39it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.39it/s]
 60%|######    | 24/40 [00:05<00:03,  4.39it/s]
 60%|######    | 24/40 [00:05<00:03,  4.39it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.40it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.40it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.37it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.37it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.38it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.38it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.39it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.39it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.38it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.38it/s]
 75%|#######5  | 30/40 [00:06<00:02,  4.39it/s]
 75%|#######5  | 30/40 [00:06<00:02,  4.39it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.38it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.38it/s]
 80%|########  | 32/40 [00:07<00:01,  4.30it/s]
 80%|########  | 32/40 [00:07<00:01,  4.30it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.33it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.33it/s]
 85%|########5 | 34/40 [00:07<00:01,  4.36it/s]
 85%|########5 | 34/40 [00:07<00:01,  4.36it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.37it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.37it/s]
 90%|######### | 36/40 [00:08<00:00,  4.37it/s]
 90%|######### | 36/40 [00:08<00:00,  4.37it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.38it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.38it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.38it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.38it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.37it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.37it/s]
+
+ - quantizing extras
+Loading transformer 2
+
+
+Quantizing Transformer 2
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
 22%|##2       | 9/40 [00:00<00:00, 89.17it/s]
 22%|##2       | 9/40 [00:00<00:00, 89.17it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.58it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.58it/s]
 68%|######7   | 27/40 [00:00<00:00, 85.76it/s]
 68%|######7   | 27/40 [00:00<00:00, 85.76it/s]
 90%|######### | 36/40 [00:00<00:00, 85.35it/s]
 90%|######### | 36/40 [00:00<00:00, 85.35it/s]
+
+ - quantizing extras
+Creating DualWanTransformer3DModel
+Loading UMT5EncoderModel
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+Using ai-toolkit/umt5_xxl_encoder for UMT5 encoder.
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing UMT5EncoderModel
+Loading VAE
+Making pipe
+Preparing Model
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 800 modules.
+enable LoRA for U-Net
+Dataset: /workspace/ai-toolkit/datasets/qwewq
+  -  Preprocessing video dimensions
+
  0%|          | 0/22 [00:00<?, ?it/s]
  0%|          | 0/22 [00:00<?, ?it/s]
+
+  -  Found 22 videos
+Bucket sizes for /workspace/ai-toolkit/datasets/qwewq:
+384x672: 11 files
+672x384: 7 files
+256x448: 1 files
+352x608: 2 files
+416x576: 1 files
+5 buckets made
+Generating baseline samples before training
+
                                     
                                     

mastur_copy/logs/0_log.txt

@@ -0,0 +1,218 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/workspace/ai-toolkit/output",
+    "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": null,
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 64,
+        "linear_alpha": 64,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 1000,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/workspace/ai-toolkit/datasets/qwewq",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "woman masturbating pussy",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 33,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "linear",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse",
+        "do_differential_guidance": true,
+        "differential_guidance_scale": 3
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "ai-toolkit/Wan2.2-T2V-A14B-Diffusers-bf16",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "wan22_14b:t2v",
+        "low_vram": false,
+        "model_kwargs": {
+            "train_high_noise": true,
+            "train_low_noise": true
+        },
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 25,
+        "num_frames": 41,
+        "fps": 16
+    }
+}
+Changing sample extention to animated webp
+Using SQLite database at /workspace/ai-toolkit/aitk_db.db
+Job ID: "83e8ed4c-6743-434a-9d8f-364c179ac62e"
+
+#############################################
+# Running job: mastur_copy
+#############################################
+
+
+Running  1 process
+Loading Wan model
+Loading transformer 1
+
+
+Quantizing Transformer 1
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
  2%|2         | 1/40 [00:00<00:11,  3.45it/s]
  2%|2         | 1/40 [00:00<00:11,  3.45it/s]
  5%|5         | 2/40 [00:00<00:09,  3.91it/s]
  5%|5         | 2/40 [00:00<00:09,  3.91it/s]
  8%|7         | 3/40 [00:00<00:09,  4.09it/s]
  8%|7         | 3/40 [00:00<00:09,  4.09it/s]
 10%|#         | 4/40 [00:00<00:08,  4.15it/s]
 10%|#         | 4/40 [00:00<00:08,  4.15it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.23it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.23it/s]
 15%|#5        | 6/40 [00:01<00:07,  4.27it/s]
 15%|#5        | 6/40 [00:01<00:07,  4.27it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.29it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.29it/s]
 20%|##        | 8/40 [00:01<00:07,  4.30it/s]
 20%|##        | 8/40 [00:01<00:07,  4.30it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.31it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.31it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.34it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.34it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.36it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.36it/s]
 30%|###       | 12/40 [00:02<00:06,  4.36it/s]
 30%|###       | 12/40 [00:02<00:06,  4.36it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.37it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.37it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.38it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.38it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.38it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.38it/s]
 40%|####      | 16/40 [00:03<00:05,  4.38it/s]
 40%|####      | 16/40 [00:03<00:05,  4.38it/s]
 42%|####2     | 17/40 [00:03<00:05,  4.39it/s]
 42%|####2     | 17/40 [00:03<00:05,  4.39it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.37it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.37it/s]
 48%|####7     | 19/40 [00:04<00:04,  4.34it/s]
 48%|####7     | 19/40 [00:04<00:04,  4.34it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.35it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.35it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.36it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.36it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.36it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.36it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.37it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.37it/s]
 60%|######    | 24/40 [00:05<00:03,  4.37it/s]
 60%|######    | 24/40 [00:05<00:03,  4.37it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.38it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.38it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.39it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.39it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.39it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.39it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.38it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.38it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.39it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.39it/s]
 75%|#######5  | 30/40 [00:06<00:02,  4.40it/s]
 75%|#######5  | 30/40 [00:06<00:02,  4.40it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.38it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.38it/s]
 80%|########  | 32/40 [00:07<00:01,  4.39it/s]
 80%|########  | 32/40 [00:07<00:01,  4.39it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.39it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.39it/s]
 85%|########5 | 34/40 [00:07<00:01,  4.37it/s]
 85%|########5 | 34/40 [00:07<00:01,  4.37it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.38it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.38it/s]
 90%|######### | 36/40 [00:08<00:00,  4.33it/s]
 90%|######### | 36/40 [00:08<00:00,  4.33it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.34it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.34it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.35it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.35it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.26it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.26it/s]
+
+ - quantizing extras
+Loading transformer 2
+
+
+Quantizing Transformer 2
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
 22%|##2       | 9/40 [00:00<00:00, 88.65it/s]
 22%|##2       | 9/40 [00:00<00:00, 88.65it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.18it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.18it/s]
 68%|######7   | 27/40 [00:00<00:00, 85.43it/s]
 68%|######7   | 27/40 [00:00<00:00, 85.43it/s]
 90%|######### | 36/40 [00:00<00:00, 85.09it/s]
 90%|######### | 36/40 [00:00<00:00, 85.09it/s]
+
+ - quantizing extras
+Creating DualWanTransformer3DModel
+Loading UMT5EncoderModel
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+Using ai-toolkit/umt5_xxl_encoder for UMT5 encoder.
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing UMT5EncoderModel
+Loading VAE
+Making pipe
+Preparing Model
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 800 modules.
+enable LoRA for U-Net
+Dataset: /workspace/ai-toolkit/datasets/qwewq
+  -  Preprocessing video dimensions
+
  0%|          | 0/22 [00:00<?, ?it/s]
  0%|          | 0/22 [00:00<?, ?it/s]
+
+  -  Found 22 videos
+Bucket sizes for /workspace/ai-toolkit/datasets/qwewq:
+384x672: 11 files
+672x384: 7 files
+256x448: 1 files
+352x608: 2 files
+416x576: 1 files
+5 buckets made
+Generating baseline samples before training
+
                                     
                                     
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2340, in run
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2340, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+

mastur_copy/logs/1_log.txt

@@ -0,0 +1,218 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/workspace/ai-toolkit/output",
+    "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": null,
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 64,
+        "linear_alpha": 64,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 1000,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/workspace/ai-toolkit/datasets/qwewq",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "woman masturbating pussy",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 41,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "linear",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse",
+        "do_differential_guidance": true,
+        "differential_guidance_scale": 3
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "ai-toolkit/Wan2.2-T2V-A14B-Diffusers-bf16",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "wan22_14b:t2v",
+        "low_vram": false,
+        "model_kwargs": {
+            "train_high_noise": true,
+            "train_low_noise": true
+        },
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 25,
+        "num_frames": 41,
+        "fps": 16
+    }
+}
+Changing sample extention to animated webp
+Using SQLite database at /workspace/ai-toolkit/aitk_db.db
+Job ID: "83e8ed4c-6743-434a-9d8f-364c179ac62e"
+
+#############################################
+# Running job: mastur_copy
+#############################################
+
+
+Running  1 process
+Loading Wan model
+Loading transformer 1
+
+
+Quantizing Transformer 1
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
  2%|2         | 1/40 [00:00<00:11,  3.38it/s]
  2%|2         | 1/40 [00:00<00:11,  3.38it/s]
  5%|5         | 2/40 [00:00<00:09,  3.85it/s]
  5%|5         | 2/40 [00:00<00:09,  3.85it/s]
  8%|7         | 3/40 [00:00<00:09,  4.06it/s]
  8%|7         | 3/40 [00:00<00:09,  4.06it/s]
 10%|#         | 4/40 [00:01<00:08,  4.03it/s]
 10%|#         | 4/40 [00:01<00:08,  4.03it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.01it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.01it/s]
 15%|#5        | 6/40 [00:01<00:08,  4.11it/s]
 15%|#5        | 6/40 [00:01<00:08,  4.11it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.17it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.17it/s]
 20%|##        | 8/40 [00:01<00:07,  4.22it/s]
 20%|##        | 8/40 [00:01<00:07,  4.22it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.26it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.26it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.29it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.29it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.31it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.31it/s]
 30%|###       | 12/40 [00:02<00:06,  4.32it/s]
 30%|###       | 12/40 [00:02<00:06,  4.32it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.34it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.34it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.36it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.36it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.36it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.36it/s]
 40%|####      | 16/40 [00:03<00:05,  4.35it/s]
 40%|####      | 16/40 [00:03<00:05,  4.35it/s]
 42%|####2     | 17/40 [00:04<00:05,  4.24it/s]
 42%|####2     | 17/40 [00:04<00:05,  4.24it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.28it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.28it/s]
 48%|####7     | 19/40 [00:04<00:05,  4.20it/s]
 48%|####7     | 19/40 [00:04<00:05,  4.20it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.24it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.24it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.21it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.21it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.26it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.26it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.30it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.30it/s]
 60%|######    | 24/40 [00:05<00:03,  4.32it/s]
 60%|######    | 24/40 [00:05<00:03,  4.32it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.34it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.34it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.35it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.35it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.36it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.36it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.34it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.34it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.18it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.18it/s]
 75%|#######5  | 30/40 [00:07<00:02,  4.22it/s]
 75%|#######5  | 30/40 [00:07<00:02,  4.22it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.22it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.22it/s]
 80%|########  | 32/40 [00:07<00:01,  4.27it/s]
 80%|########  | 32/40 [00:07<00:01,  4.27it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.31it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.31it/s]
 85%|########5 | 34/40 [00:08<00:01,  4.34it/s]
 85%|########5 | 34/40 [00:08<00:01,  4.34it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.32it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.32it/s]
 90%|######### | 36/40 [00:08<00:00,  4.33it/s]
 90%|######### | 36/40 [00:08<00:00,  4.33it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.31it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.31it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.31it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.31it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.22it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.22it/s]
+
+ - quantizing extras
+Loading transformer 2
+
+
+Quantizing Transformer 2
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
 22%|##2       | 9/40 [00:00<00:00, 88.63it/s]
 22%|##2       | 9/40 [00:00<00:00, 88.63it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.26it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.26it/s]
 68%|######7   | 27/40 [00:00<00:00, 85.49it/s]
 68%|######7   | 27/40 [00:00<00:00, 85.49it/s]
 90%|######### | 36/40 [00:00<00:00, 85.20it/s]
 90%|######### | 36/40 [00:00<00:00, 85.20it/s]
+
+ - quantizing extras
+Creating DualWanTransformer3DModel
+Loading UMT5EncoderModel
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+Using ai-toolkit/umt5_xxl_encoder for UMT5 encoder.
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing UMT5EncoderModel
+Loading VAE
+Making pipe
+Preparing Model
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 800 modules.
+enable LoRA for U-Net
+Dataset: /workspace/ai-toolkit/datasets/qwewq
+  -  Preprocessing video dimensions
+
  0%|          | 0/22 [00:00<?, ?it/s]
  0%|          | 0/22 [00:00<?, ?it/s]
+
+  -  Found 22 videos
+Bucket sizes for /workspace/ai-toolkit/datasets/qwewq:
+384x672: 11 files
+672x384: 7 files
+256x448: 1 files
+352x608: 2 files
+416x576: 1 files
+5 buckets made
+Generating baseline samples before training
+
                                     
                                     
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2340, in run
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2340, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+

mastur_copy/logs/2_log.txt

@@ -0,0 +1,218 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/workspace/ai-toolkit/output",
+    "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": null,
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 64,
+        "linear_alpha": 64,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 1000,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/workspace/ai-toolkit/datasets/qwewq",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "woman masturbating pussy",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 81,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "linear",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse",
+        "do_differential_guidance": true,
+        "differential_guidance_scale": 3
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "ai-toolkit/Wan2.2-T2V-A14B-Diffusers-bf16",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "wan22_14b:t2v",
+        "low_vram": false,
+        "model_kwargs": {
+            "train_high_noise": true,
+            "train_low_noise": true
+        },
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 25,
+        "num_frames": 41,
+        "fps": 16
+    }
+}
+Changing sample extention to animated webp
+Using SQLite database at /workspace/ai-toolkit/aitk_db.db
+Job ID: "83e8ed4c-6743-434a-9d8f-364c179ac62e"
+
+#############################################
+# Running job: mastur_copy
+#############################################
+
+
+Running  1 process
+Loading Wan model
+Loading transformer 1
+
+
+Quantizing Transformer 1
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
  2%|2         | 1/40 [00:00<00:11,  3.38it/s]
  2%|2         | 1/40 [00:00<00:11,  3.38it/s]
  5%|5         | 2/40 [00:00<00:09,  3.81it/s]
  5%|5         | 2/40 [00:00<00:09,  3.81it/s]
  8%|7         | 3/40 [00:00<00:09,  4.02it/s]
  8%|7         | 3/40 [00:00<00:09,  4.02it/s]
 10%|#         | 4/40 [00:00<00:08,  4.14it/s]
 10%|#         | 4/40 [00:00<00:08,  4.14it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.23it/s]
 12%|#2        | 5/40 [00:01<00:08,  4.23it/s]
 15%|#5        | 6/40 [00:01<00:07,  4.26it/s]
 15%|#5        | 6/40 [00:01<00:07,  4.26it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.29it/s]
 18%|#7        | 7/40 [00:01<00:07,  4.29it/s]
 20%|##        | 8/40 [00:01<00:07,  4.31it/s]
 20%|##        | 8/40 [00:01<00:07,  4.31it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.33it/s]
 22%|##2       | 9/40 [00:02<00:07,  4.33it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.35it/s]
 25%|##5       | 10/40 [00:02<00:06,  4.35it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.37it/s]
 28%|##7       | 11/40 [00:02<00:06,  4.37it/s]
 30%|###       | 12/40 [00:02<00:06,  4.38it/s]
 30%|###       | 12/40 [00:02<00:06,  4.38it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.39it/s]
 32%|###2      | 13/40 [00:03<00:06,  4.39it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.38it/s]
 35%|###5      | 14/40 [00:03<00:05,  4.38it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.38it/s]
 38%|###7      | 15/40 [00:03<00:05,  4.38it/s]
 40%|####      | 16/40 [00:03<00:05,  4.39it/s]
 40%|####      | 16/40 [00:03<00:05,  4.39it/s]
 42%|####2     | 17/40 [00:03<00:05,  4.38it/s]
 42%|####2     | 17/40 [00:03<00:05,  4.38it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.39it/s]
 45%|####5     | 18/40 [00:04<00:05,  4.39it/s]
 48%|####7     | 19/40 [00:04<00:04,  4.40it/s]
 48%|####7     | 19/40 [00:04<00:04,  4.40it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.38it/s]
 50%|#####     | 20/40 [00:04<00:04,  4.38it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.39it/s]
 52%|#####2    | 21/40 [00:04<00:04,  4.39it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.39it/s]
 55%|#####5    | 22/40 [00:05<00:04,  4.39it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.39it/s]
 57%|#####7    | 23/40 [00:05<00:03,  4.39it/s]
 60%|######    | 24/40 [00:05<00:03,  4.39it/s]
 60%|######    | 24/40 [00:05<00:03,  4.39it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.40it/s]
 62%|######2   | 25/40 [00:05<00:03,  4.40it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.40it/s]
 65%|######5   | 26/40 [00:06<00:03,  4.40it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.40it/s]
 68%|######7   | 27/40 [00:06<00:02,  4.40it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.40it/s]
 70%|#######   | 28/40 [00:06<00:02,  4.40it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.39it/s]
 72%|#######2  | 29/40 [00:06<00:02,  4.39it/s]
 75%|#######5  | 30/40 [00:06<00:02,  4.41it/s]
 75%|#######5  | 30/40 [00:06<00:02,  4.41it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.41it/s]
 78%|#######7  | 31/40 [00:07<00:02,  4.41it/s]
 80%|########  | 32/40 [00:07<00:01,  4.24it/s]
 80%|########  | 32/40 [00:07<00:01,  4.24it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.20it/s]
 82%|########2 | 33/40 [00:07<00:01,  4.20it/s]
 85%|########5 | 34/40 [00:07<00:01,  4.27it/s]
 85%|########5 | 34/40 [00:07<00:01,  4.27it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.31it/s]
 88%|########7 | 35/40 [00:08<00:01,  4.31it/s]
 90%|######### | 36/40 [00:08<00:00,  4.34it/s]
 90%|######### | 36/40 [00:08<00:00,  4.34it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.37it/s]
 92%|#########2| 37/40 [00:08<00:00,  4.37it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.38it/s]
 95%|#########5| 38/40 [00:08<00:00,  4.38it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.40it/s]
 98%|#########7| 39/40 [00:09<00:00,  4.40it/s]
+
+ - quantizing extras
+Loading transformer 2
+
+
+Quantizing Transformer 2
+ - quantizing 40 transformer blocks
+
  0%|          | 0/40 [00:00<?, ?it/s]
  0%|          | 0/40 [00:00<?, ?it/s]
 22%|##2       | 9/40 [00:00<00:00, 88.97it/s]
 22%|##2       | 9/40 [00:00<00:00, 88.97it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.37it/s]
 45%|####5     | 18/40 [00:00<00:00, 86.37it/s]
 68%|######7   | 27/40 [00:00<00:00, 84.32it/s]
 68%|######7   | 27/40 [00:00<00:00, 84.32it/s]
 90%|######### | 36/40 [00:00<00:00, 83.41it/s]
 90%|######### | 36/40 [00:00<00:00, 83.41it/s]
+
+ - quantizing extras
+Creating DualWanTransformer3DModel
+Loading UMT5EncoderModel
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+Using ai-toolkit/umt5_xxl_encoder for UMT5 encoder.
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing UMT5EncoderModel
+Loading VAE
+Making pipe
+Preparing Model
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 800 modules.
+enable LoRA for U-Net
+Dataset: /workspace/ai-toolkit/datasets/qwewq
+  -  Preprocessing video dimensions
+
  0%|          | 0/22 [00:00<?, ?it/s]
  0%|          | 0/22 [00:00<?, ?it/s]
+
+  -  Found 22 videos
+Bucket sizes for /workspace/ai-toolkit/datasets/qwewq:
+384x672: 11 files
+672x384: 7 files
+256x448: 1 files
+352x608: 2 files
+416x576: 1 files
+5 buckets made
+Generating baseline samples before training
+
                                     
                                     
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2340, in run
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2340, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+

mastur_copy/pid.txt

@@ -0,0 +1 @@
+56970