Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,37 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+samples/1755076508263__000000000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755076886864__000000000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755078728284__000000250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755080569525__000000500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755082333508__000000750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755084158527__000001000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755085871227__000001250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755089492165__000001500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755091316227__000001750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755093096161__000002000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755094934349__000002250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755096711970__000002500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755098476620__000002750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755100277602__000003000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755102077128__000003250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755103880705__000003500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755105669238__000003750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755107447385__000004000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755109230437__000004250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755111038123__000004500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755112806995__000004750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755114575647__000005000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755116365454__000005250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755118191909__000005500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755119974698__000005750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755121739761__000006000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755123468036__000006250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755125240082__000006500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755127010957__000006750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755128849859__000007000_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755130588958__000007250_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755132336218__000007500_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755134142831__000007750_0.webp filter=lfs diff=lfs merge=lfs -text
+samples/1755135913723__000008000_0.webp filter=lfs diff=lfs merge=lfs -text

.job_config.json

@@ -0,0 +1,118 @@
+{
+  "job": "extension",
+  "config": {
+    "name": "Ineffa_wan_5B_lora_v1",
+    "process": [
+      {
+        "type": "ui_trainer",
+        "training_folder": "/home/featurize/ai-toolkit/output",
+        "sqlite_db_path": "/home/featurize/ai-toolkit/aitk_db.db",
+        "device": "cuda",
+        "trigger_word": null,
+        "performance_log_every": 10,
+        "network": {
+          "type": "lora",
+          "linear": 16,
+          "linear_alpha": 16,
+          "conv": 16,
+          "conv_alpha": 16,
+          "lokr_full_rank": true,
+          "lokr_factor": -1,
+          "network_kwargs": {
+            "ignore_if_contains": []
+          }
+        },
+        "save": {
+          "dtype": "bf16",
+          "save_every": 250,
+          "max_step_saves_to_keep": 4000,
+          "save_format": "diffusers",
+          "push_to_hub": false
+        },
+        "datasets": [
+          {
+            "folder_path": "/home/featurize/ai-toolkit/datasets/Ineffa_videos_captioned_960x544x4_9",
+            "control_path": null,
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+              512,
+              768
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 49,
+            "do_i2v": true
+          }
+        ],
+        "train": {
+          "batch_size": 1,
+          "bypass_guidance_embedding": false,
+          "steps": 30000,
+          "gradient_accumulation": 1,
+          "train_unet": true,
+          "train_text_encoder": false,
+          "gradient_checkpointing": true,
+          "noise_scheduler": "flowmatch",
+          "optimizer": "adamw8bit",
+          "timestep_type": "weighted",
+          "content_or_style": "balanced",
+          "optimizer_params": {
+            "weight_decay": 0.0001
+          },
+          "unload_text_encoder": false,
+          "cache_text_embeddings": false,
+          "lr": 0.0001,
+          "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+          },
+          "skip_first_sample": false,
+          "disable_sampling": false,
+          "dtype": "bf16",
+          "diff_output_preservation": false,
+          "diff_output_preservation_multiplier": 1,
+          "diff_output_preservation_class": "person"
+        },
+        "model": {
+          "name_or_path": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+          "quantize": true,
+          "qtype": "qfloat8",
+          "quantize_te": true,
+          "qtype_te": "qfloat8",
+          "arch": "wan22_5b",
+          "low_vram": true,
+          "model_kwargs": {}
+        },
+        "sample": {
+          "sampler": "flowmatch",
+          "sample_every": 250,
+          "width": 768,
+          "height": 768,
+          "samples": [
+            {
+              "prompt": "The video features a character with pale blue hair and striking blue eyes. The character is adorned in a detailed, futuristic outfit that includes a  dark blue and gold color scheme. The attire consists of a fitted bodysuit with metallic accents, gloves, and boots, all embellished with intricate golden designs and symbols.   The character eat an icecream."
+            }
+          ],
+          "neg": "",
+          "seed": 42,
+          "walk_seed": true,
+          "guidance_scale": 4,
+          "sample_steps": 25,
+          "num_frames": 49,
+          "fps": 24
+        }
+      }
+    ]
+  },
+  "meta": {
+    "name": "[name]",
+    "version": "1.0"
+  }
+}

Ineffa_wan_5B_lora_v1_000000250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23b6f38773e2e37b18c815b521ccdc9892ed8d472395da305ade4796250dbff7
+size 80683336

Ineffa_wan_5B_lora_v1_000000500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6c59149441f70e946d5f2e0b826cbeb5be3a1cfb55b50e4ef608fbe52c2fb4b
+size 80683336

Ineffa_wan_5B_lora_v1_000000750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a10c17e36d82d3e9e5cd21908289eca48714565812f27f67e5ba3fc4e524947f
+size 80683336

Ineffa_wan_5B_lora_v1_000001000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962bbf95a26585d8366aee97180fe43aa6458b6c66bebaea85c6e2fa93af88f0
+size 80683336

Ineffa_wan_5B_lora_v1_000001250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcdb7e7a8ac661c80bde8b84f7da0db05a47e39e9f2735340d231c7d3adb101b
+size 80683336

Ineffa_wan_5B_lora_v1_000001500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5938036ab0f6480702192267fd05672a083901fb75227f577b37700bad7ed4f
+size 80683336

Ineffa_wan_5B_lora_v1_000001750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a714c13430fd39922bdaf2b1b425fbef074d3c3dcad12bb57be007426ca8f88
+size 80683336

Ineffa_wan_5B_lora_v1_000002000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8924fc53ebd5fcd3c121b0ad7df9e157f75d3c79f6acdb5b152ae8a9702b52d9
+size 80683336

Ineffa_wan_5B_lora_v1_000002250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2687d4b7b345c5c5809abc228ab2c7d2840eade5cefa6e387c20529d0e701318
+size 80683336

Ineffa_wan_5B_lora_v1_000002500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd42814e02c7c7f035428e06d87486a9b834c0ff239860ece45ac403a6b7bc4f
+size 80683336

Ineffa_wan_5B_lora_v1_000002750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d914ee6c9a1d7e896c0d1ccbecd5b0d1c3bbb19ea0a730d2c3d1cf3ac06e12ab
+size 80683336

Ineffa_wan_5B_lora_v1_000003000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9479d034f8fee2731da1bbf27880fb21107a4d12e2cf7a940637ed9c1a6e38c
+size 80683336

Ineffa_wan_5B_lora_v1_000003250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43f2e6a1bc3c56eb8f3668b047c3dfe7a3ecf5e611dd9e2b40ac6ff1a34ecd13
+size 80683336

Ineffa_wan_5B_lora_v1_000003500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a25500e5275d0a6048dbd2c1eb6284f42fa55e264b780eef563720449d63649d
+size 80683336

Ineffa_wan_5B_lora_v1_000003750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4007268933d8879ce09bc76201aa9d5a83f09125510f3fed5315839561e71208
+size 80683336

Ineffa_wan_5B_lora_v1_000004000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13a1ff922329168c50fc4ae474eb0b0d3ebcf6b80048e29cbca28d08d07a2892
+size 80683336

Ineffa_wan_5B_lora_v1_000004250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aef64478ec7c94d5dec1a626245594ae202a083aa925e410dbc1fb16e9f7c060
+size 80683336

Ineffa_wan_5B_lora_v1_000004500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8cea76de12aaea1ea58afc86ff153c696879894b28079be9a7bec048227ff8c
+size 80683336

Ineffa_wan_5B_lora_v1_000004750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbe2ad1c62a610de66b726242004756a615bd3e351c9e81afbd47fe3bc019774
+size 80683336

Ineffa_wan_5B_lora_v1_000005000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06be88f66427f78ab0dfd9de533a1c0ac78e6b723314e3969cd319deaa8ef248
+size 80683336

Ineffa_wan_5B_lora_v1_000005250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb75ada83a560ffd85f659036a565230268dc769101f9a858fe3fb267c3eb6d7
+size 80683336

Ineffa_wan_5B_lora_v1_000005500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d87350a548d0a7dad364bf311477e1964dc086580a6c298184a896a9542e1a2
+size 80683336

Ineffa_wan_5B_lora_v1_000005750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57f652d8cbd85dcacbd416235f0a603751c922a305f7d121ff7f0f43e55fe38d
+size 80683336

Ineffa_wan_5B_lora_v1_000006000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f5858543c62160b37b1d8f9aff08022c67caf43efb70c11813833ed05049865
+size 80683336

Ineffa_wan_5B_lora_v1_000006250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5dd0597ed9d6e4ca35ce27edab14cd4f1e3d5888cdd4a09f65022a15f28c664
+size 80683336

Ineffa_wan_5B_lora_v1_000006500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:418c4145133eafb509f274adf887f7b676d2df97353572848072a5dd13ec61da
+size 80683336

Ineffa_wan_5B_lora_v1_000006750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d15d34e51ae42397b704bd4dfe4b8972b4bb0c9d044f9ea3d4cd632ebc296da
+size 80683336

Ineffa_wan_5B_lora_v1_000007000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad8a1c5877e267c51a95ea700d14fa246e5a935c2c27f4557f0b4aef3dcc2765
+size 80683336

Ineffa_wan_5B_lora_v1_000007250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f991b5dfbd60d52b07c0180ccf75a9977f81bab026b5e96cb4bee5e032ec8dc0
+size 80683336

Ineffa_wan_5B_lora_v1_000007500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99041bdc4abd1d47a051b8e4be2e1d90e5418eb98089ec30bed12dcdc40b5f62
+size 80683336

Ineffa_wan_5B_lora_v1_000007750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f703ef0aaa3f45ac192ecb2a9493ad2241cda7cd36373f849be0dc457c6c128
+size 80683336

Ineffa_wan_5B_lora_v1_000008000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6161f60f3ae57cc47a04714288fbce5322466fd4c865b919990819a2bbce4c4
+size 80683336

config.yaml

@@ -0,0 +1,100 @@
+job: extension
+config:
+  name: Ineffa_wan_5B_lora_v1
+  process:
+  - type: ui_trainer
+    training_folder: /home/featurize/ai-toolkit/output
+    sqlite_db_path: /home/featurize/ai-toolkit/aitk_db.db
+    device: cuda
+    trigger_word: null
+    performance_log_every: 10
+    network:
+      type: lora
+      linear: 16
+      linear_alpha: 16
+      conv: 16
+      conv_alpha: 16
+      lokr_full_rank: true
+      lokr_factor: -1
+      network_kwargs:
+        ignore_if_contains: []
+    save:
+      dtype: bf16
+      save_every: 250
+      max_step_saves_to_keep: 4000
+      save_format: diffusers
+      push_to_hub: false
+    datasets:
+    - folder_path: /home/featurize/ai-toolkit/datasets/Ineffa_videos_captioned_960x544x4_9
+      control_path: null
+      mask_path: null
+      mask_min_value: 0.1
+      default_caption: ''
+      caption_ext: txt
+      caption_dropout_rate: 0.05
+      cache_latents_to_disk: false
+      is_reg: false
+      network_weight: 1
+      resolution:
+      - 512
+      - 768
+      controls: []
+      shrink_video_to_frames: true
+      num_frames: 49
+      do_i2v: true
+    train:
+      batch_size: 1
+      bypass_guidance_embedding: false
+      steps: 30000
+      gradient_accumulation: 1
+      train_unet: true
+      train_text_encoder: false
+      gradient_checkpointing: true
+      noise_scheduler: flowmatch
+      optimizer: adamw8bit
+      timestep_type: weighted
+      content_or_style: balanced
+      optimizer_params:
+        weight_decay: 0.0001
+      unload_text_encoder: false
+      cache_text_embeddings: false
+      lr: 0.0001
+      ema_config:
+        use_ema: false
+        ema_decay: 0.99
+      skip_first_sample: false
+      disable_sampling: false
+      dtype: bf16
+      diff_output_preservation: false
+      diff_output_preservation_multiplier: 1
+      diff_output_preservation_class: person
+    model:
+      name_or_path: Wan-AI/Wan2.2-TI2V-5B-Diffusers
+      quantize: true
+      qtype: qfloat8
+      quantize_te: true
+      qtype_te: qfloat8
+      arch: wan22_5b
+      low_vram: true
+      model_kwargs: {}
+    sample:
+      sampler: flowmatch
+      sample_every: 250
+      width: 768
+      height: 768
+      samples:
+      - prompt: The video features a character with pale blue hair and striking blue
+          eyes. The character is adorned in a detailed, futuristic outfit that includes
+          a  dark blue and gold color scheme. The attire consists of a fitted bodysuit
+          with metallic accents, gloves, and boots, all embellished with intricate
+          golden designs and symbols.   The character eat an icecream.
+      neg: ''
+      seed: 42
+      walk_seed: true
+      guidance_scale: 4
+      sample_steps: 25
+      num_frames: 49
+      fps: 24
+meta:
+  name: Ineffa_wan_5B_lora_v1
+  version: '1.0'

logs/1_log.txt

@@ -0,0 +1,260 @@
+Running 1 job
+{
+    "type": "ui_trainer",
+    "training_folder": "/home/featurize/ai-toolkit/output",
+    "sqlite_db_path": "/home/featurize/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": null,
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 16,
+        "linear_alpha": 16,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 250,
+        "max_step_saves_to_keep": 4000,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/home/featurize/ai-toolkit/datasets/Ineffa_videos_captioned_960x544x4",
+            "control_path": null,
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512,
+                768
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 33,
+            "do_i2v": true
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 30000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person"
+    },
+    "model": {
+        "name_or_path": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "wan22_5b",
+        "low_vram": true,
+        "model_kwargs": {}
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 768,
+        "height": 768,
+        "samples": [
+            {
+                "prompt": "The video features a character with pale blue hair and striking blue eyes. The character is adorned in a detailed, futuristic outfit that includes a  dark blue and gold color scheme. The attire consists of a fitted bodysuit with metallic accents, gloves, and boots, all embellished with intricate golden designs and symbols.   The character eat an icecream."
+            }
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 25,
+        "num_frames": 33,
+        "fps": 24
+    }
+}
+Using EMA
+Changing sample extention to animated webp
+Using SQLite database at /home/featurize/ai-toolkit/aitk_db.db
+Job ID: "42872fb6-b292-4972-872e-3f11fad398bb"
+
+#############################################
+# Running job: Ineffa_wan_5B_lora_v1
+#############################################
+
+
+Running  1 process
+Loading Wan model
+Loading transformer
+
+
+
+
+Quantizing Transformer
+Quantizing blocks
+
  0%|                                                                                                                                                  | 0/30 [00:00<?, ?it/s]
  0%|                                                                                                                                                  | 0/30 [00:00<?, ?it/s]
  3%|####6                                                                                                                                     | 1/30 [00:00<00:16,  1.76it/s]
  3%|####6                                                                                                                                     | 1/30 [00:00<00:16,  1.76it/s]
  7%|#########2                                                                                                                                | 2/30 [00:00<00:12,  2.21it/s]
  7%|#########2                                                                                                                                | 2/30 [00:00<00:12,  2.21it/s]
 10%|#############8                                                                                                                            | 3/30 [00:01<00:11,  2.43it/s]
 10%|#############8                                                                                                                            | 3/30 [00:01<00:11,  2.43it/s]
 13%|##################4                                                                                                                       | 4/30 [00:01<00:10,  2.54it/s]
 13%|##################4                                                                                                                       | 4/30 [00:01<00:10,  2.54it/s]
 17%|#######################                                                                                                                   | 5/30 [00:02<00:09,  2.61it/s]
 17%|#######################                                                                                                                   | 5/30 [00:02<00:09,  2.61it/s]
 20%|###########################6                                                                                                              | 6/30 [00:02<00:09,  2.66it/s]
 20%|###########################6                                                                                                              | 6/30 [00:02<00:09,  2.66it/s]
 23%|################################2                                                                                                         | 7/30 [00:02<00:08,  2.69it/s]
 23%|################################2                                                                                                         | 7/30 [00:02<00:08,  2.69it/s]
 27%|####################################8                                                                                                     | 8/30 [00:03<00:08,  2.72it/s]
 27%|####################################8                                                                                                     | 8/30 [00:03<00:08,  2.72it/s]
 30%|#########################################4                                                                                                | 9/30 [00:03<00:07,  2.74it/s]
 30%|#########################################4                                                                                                | 9/30 [00:03<00:07,  2.74it/s]
 33%|#############################################6                                                                                           | 10/30 [00:03<00:07,  2.76it/s]
 33%|#############################################6                                                                                           | 10/30 [00:03<00:07,  2.76it/s]
 37%|##################################################2                                                                                      | 11/30 [00:04<00:06,  2.77it/s]
 37%|##################################################2                                                                                      | 11/30 [00:04<00:06,  2.77it/s]
 40%|######################################################8                                                                                  | 12/30 [00:04<00:06,  2.77it/s]
 40%|######################################################8                                                                                  | 12/30 [00:04<00:06,  2.77it/s]
 43%|###########################################################3                                                                             | 13/30 [00:04<00:06,  2.78it/s]
 43%|###########################################################3                                                                             | 13/30 [00:04<00:06,  2.78it/s]
 47%|###############################################################9                                                                         | 14/30 [00:05<00:05,  2.79it/s]
 47%|###############################################################9                                                                         | 14/30 [00:05<00:05,  2.79it/s]
 50%|####################################################################5                                                                    | 15/30 [00:05<00:05,  2.80it/s]
 50%|####################################################################5                                                                    | 15/30 [00:05<00:05,  2.80it/s]
 53%|#########################################################################                                                                | 16/30 [00:05<00:04,  2.80it/s]
 53%|#########################################################################                                                                | 16/30 [00:05<00:04,  2.80it/s]
 57%|#############################################################################6                                                           | 17/30 [00:06<00:04,  2.80it/s]
 57%|#############################################################################6                                                           | 17/30 [00:06<00:04,  2.80it/s]
 60%|##################################################################################2                                                      | 18/30 [00:06<00:04,  2.77it/s]
 60%|##################################################################################2                                                      | 18/30 [00:06<00:04,  2.77it/s]
 63%|######################################################################################7                                                  | 19/30 [00:07<00:03,  2.77it/s]
 63%|######################################################################################7                                                  | 19/30 [00:07<00:03,  2.77it/s]
 67%|###########################################################################################3                                             | 20/30 [00:07<00:04,  2.44it/s]
 67%|###########################################################################################3                                             | 20/30 [00:07<00:04,  2.44it/s]
 70%|###############################################################################################8                                         | 21/30 [00:07<00:03,  2.48it/s]
 70%|###############################################################################################8                                         | 21/30 [00:07<00:03,  2.48it/s]
 73%|####################################################################################################4                                    | 22/30 [00:08<00:03,  2.54it/s]
 73%|####################################################################################################4                                    | 22/30 [00:08<00:03,  2.54it/s]
 77%|#########################################################################################################                                | 23/30 [00:08<00:02,  2.59it/s]
 77%|#########################################################################################################                                | 23/30 [00:08<00:02,  2.59it/s]
 80%|#############################################################################################################6                           | 24/30 [00:09<00:02,  2.64it/s]
 80%|#############################################################################################################6                           | 24/30 [00:09<00:02,  2.64it/s]
 83%|##################################################################################################################1                      | 25/30 [00:09<00:01,  2.67it/s]
 83%|##################################################################################################################1                      | 25/30 [00:09<00:01,  2.67it/s]
 87%|######################################################################################################################7                  | 26/30 [00:09<00:01,  2.70it/s]
 87%|######################################################################################################################7                  | 26/30 [00:09<00:01,  2.70it/s]
 90%|###########################################################################################################################3             | 27/30 [00:10<00:01,  2.70it/s]
 90%|###########################################################################################################################3             | 27/30 [00:10<00:01,  2.70it/s]
 93%|###############################################################################################################################8         | 28/30 [00:10<00:00,  2.70it/s]
 93%|###############################################################################################################################8         | 28/30 [00:10<00:00,  2.70it/s]
 97%|####################################################################################################################################4    | 29/30 [00:10<00:00,  2.66it/s]
 97%|####################################################################################################################################4    | 29/30 [00:10<00:00,  2.66it/s]
+
+Quantizing the rest
+Loading UMT5EncoderModel
+
+
+Quantizing UMT5EncoderModel
+Moving transformer back to GPU
+Loading VAE
+Making pipe
+Preparing Model
+create LoRA network. base dim (rank): 16, alpha: 16
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 300 modules.
+enable LoRA for U-Net
+Dataset: /home/featurize/ai-toolkit/datasets/Ineffa_videos_captioned_960x544x4
+  -  Preprocessing video dimensions
+
  0%|                                                                                                                                                 | 0/918 [00:00<?, ?it/s]
  0%|                                                                                                                                                 | 0/918 [00:00<?, ?it/s]
+
+  -  Found 918 videos
+Bucket sizes for /home/featurize/ai-toolkit/datasets/Ineffa_videos_captioned_960x544x4:
+672x384: 918 files
+1 buckets made
+Dataset: /home/featurize/ai-toolkit/datasets/Ineffa_videos_captioned_960x544x4
+  -  Preprocessing video dimensions
+
  0%|                                                                                                                                                 | 0/918 [00:00<?, ?it/s]
  0%|                                                                                                                                                 | 0/918 [00:00<?, ?it/s]
+
+  -  Found 918 videos
+Bucket sizes for /home/featurize/ai-toolkit/datasets/Ineffa_videos_captioned_960x544x4:
+896x512: 918 files
+1 buckets made
+Generating baseline samples before training
+
+Unloading text encoder
+
+
+
  0%|                                                                                                                                                  | 0/25 [00:00<?, ?it/s]
  0%|                                                                                                                                                  | 0/25 [00:00<?, ?it/s]
+
+
  4%|#####5                                                                                                                                    | 1/25 [00:03<01:20,  3.34s/it]
  4%|#####5                                                                                                                                    | 1/25 [00:03<01:20,  3.34s/it]
+
+
  8%|###########                                                                                                                               | 2/25 [00:06<01:12,  3.14s/it]
  8%|###########                                                                                                                               | 2/25 [00:06<01:12,  3.14s/it]
+
+
 12%|################5                                                                                                                         | 3/25 [00:09<01:08,  3.10s/it]
 12%|################5                                                                                                                         | 3/25 [00:09<01:08,  3.10s/it]
+
+
 16%|######################                                                                                                                    | 4/25 [00:12<01:04,  3.06s/it]
 16%|######################                                                                                                                    | 4/25 [00:12<01:04,  3.06s/it]
+
+
 20%|###########################6                                                                                                              | 5/25 [00:15<01:00,  3.04s/it]
 20%|###########################6                                                                                                              | 5/25 [00:15<01:00,  3.04s/it]
+
+
 24%|#################################1                                                                                                        | 6/25 [00:18<00:57,  3.03s/it]
 24%|#################################1                                                                                                        | 6/25 [00:18<00:57,  3.03s/it]
+
+
 28%|######################################6                                                                                                   | 7/25 [00:21<00:54,  3.02s/it]
 28%|######################################6                                                                                                   | 7/25 [00:21<00:54,  3.02s/it]
+
+
 32%|############################################1                                                                                             | 8/25 [00:24<00:51,  3.02s/it]
 32%|############################################1                                                                                             | 8/25 [00:24<00:51,  3.02s/it]
+
+
 36%|#################################################6                                                                                        | 9/25 [00:27<00:48,  3.02s/it]
 36%|#################################################6                                                                                        | 9/25 [00:27<00:48,  3.02s/it]
+
+
 40%|######################################################8                                                                                  | 10/25 [00:30<00:45,  3.01s/it]
 40%|######################################################8                                                                                  | 10/25 [00:30<00:45,  3.01s/it]
+
+
 44%|############################################################2                                                                            | 11/25 [00:33<00:42,  3.01s/it]
 44%|############################################################2                                                                            | 11/25 [00:33<00:42,  3.01s/it]
+
+
 48%|#################################################################7                                                                       | 12/25 [00:36<00:39,  3.01s/it]
 48%|#################################################################7                                                                       | 12/25 [00:36<00:39,  3.01s/it]
+
+
 52%|#######################################################################2                                                                 | 13/25 [00:39<00:36,  3.01s/it]
 52%|#######################################################################2                                                                 | 13/25 [00:39<00:36,  3.01s/it]
+
+
 56%|############################################################################7                                                            | 14/25 [00:42<00:33,  3.01s/it]
 56%|############################################################################7                                                            | 14/25 [00:42<00:33,  3.01s/it]
+
+
 60%|##################################################################################2                                                      | 15/25 [00:45<00:30,  3.02s/it]
 60%|##################################################################################2                                                      | 15/25 [00:45<00:30,  3.02s/it]
+
+
 64%|#######################################################################################6                                                 | 16/25 [00:48<00:27,  3.02s/it]
 64%|#######################################################################################6                                                 | 16/25 [00:48<00:27,  3.02s/it]
+
+
 68%|#############################################################################################1                                           | 17/25 [00:51<00:24,  3.02s/it]
 68%|#############################################################################################1                                           | 17/25 [00:51<00:24,  3.02s/it]
+
+
 72%|##################################################################################################6                                      | 18/25 [00:54<00:21,  3.02s/it]
 72%|##################################################################################################6                                      | 18/25 [00:54<00:21,  3.02s/it]
+
+
 76%|########################################################################################################1                                | 19/25 [00:57<00:18,  3.03s/it]
 76%|########################################################################################################1                                | 19/25 [00:57<00:18,  3.03s/it]
+
+
 80%|#############################################################################################################6                           | 20/25 [01:00<00:15,  3.03s/it]
 80%|#############################################################################################################6                           | 20/25 [01:00<00:15,  3.03s/it]
+
+
 84%|###################################################################################################################                      | 21/25 [01:03<00:12,  3.04s/it]
 84%|###################################################################################################################                      | 21/25 [01:03<00:12,  3.04s/it]
+
+
 88%|########################################################################################################################5                | 22/25 [01:06<00:09,  3.05s/it]
 88%|########################################################################################################################5                | 22/25 [01:06<00:09,  3.05s/it]
+
+
 92%|##############################################################################################################################           | 23/25 [01:09<00:06,  3.07s/it]
 92%|##############################################################################################################################           | 23/25 [01:09<00:06,  3.07s/it]
+
+
 96%|###################################################################################################################################5     | 24/25 [01:13<00:03,  3.08s/it]
 96%|###################################################################################################################################5     | 24/25 [01:13<00:03,  3.08s/it]
+
+
+
+Unloading transformer
+Loading Vae
+Moving transformer back to device
+
                                                                                                                                                                              
                                                                                                                                                                              
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/home/featurize/ai-toolkit/run.py", line 120, in <module>
+  File "/home/featurize/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/home/featurize/ai-toolkit/run.py", line 108, in main
+  File "/home/featurize/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/home/featurize/ai-toolkit/run.py", line 96, in main
+  File "/home/featurize/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2248, in run
+  File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2248, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 188, in end_step_hook
+  File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 188, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 79, in maybe_stop
+  File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 79, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f75f82fffe7f3050a43a142ba4c8c55d6c54a057fb163a3ee0d8e87e5e2da819
+size 82570949