Upload folder using huggingface_hub

.job_config.json

@@ -0,0 +1,119 @@
+{
+  "job": "extension",
+  "config": {
+    "name": "qwen_image_sethos_lora_v1",
+    "process": [
+      {
+        "type": "ui_trainer",
+        "training_folder": "/home/featurize/ai-toolkit/output",
+        "sqlite_db_path": "/home/featurize/ai-toolkit/aitk_db.db",
+        "device": "cuda",
+        "trigger_word": null,
+        "performance_log_every": 10,
+        "network": {
+          "type": "lora",
+          "linear": 16,
+          "linear_alpha": 16,
+          "conv": 16,
+          "conv_alpha": 16,
+          "lokr_full_rank": true,
+          "lokr_factor": -1,
+          "network_kwargs": {
+            "ignore_if_contains": []
+          }
+        },
+        "save": {
+          "dtype": "bf16",
+          "save_every": 250,
+          "max_step_saves_to_keep": 4000,
+          "save_format": "diffusers",
+          "push_to_hub": false
+        },
+        "datasets": [
+          {
+            "folder_path": "/home/featurize/ai-toolkit/datasets/Sethos_Images_Captioned",
+            "control_path": null,
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+              512,
+              768,
+              1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "do_i2v": true
+          }
+        ],
+        "train": {
+          "batch_size": 1,
+          "bypass_guidance_embedding": false,
+          "steps": 30000,
+          "gradient_accumulation": 1,
+          "train_unet": true,
+          "train_text_encoder": false,
+          "gradient_checkpointing": true,
+          "noise_scheduler": "flowmatch",
+          "optimizer": "adamw8bit",
+          "timestep_type": "weighted",
+          "content_or_style": "balanced",
+          "optimizer_params": {
+            "weight_decay": 0.0001
+          },
+          "unload_text_encoder": false,
+          "cache_text_embeddings": true,
+          "lr": 0.0001,
+          "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+          },
+          "skip_first_sample": false,
+          "disable_sampling": false,
+          "dtype": "bf16",
+          "diff_output_preservation": false,
+          "diff_output_preservation_multiplier": 1,
+          "diff_output_preservation_class": "person"
+        },
+        "model": {
+          "name_or_path": "Qwen/Qwen-Image",
+          "quantize": true,
+          "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_torchao_uint3.safetensors",
+          "quantize_te": true,
+          "qtype_te": "qfloat8",
+          "arch": "qwen_image",
+          "low_vram": true,
+          "model_kwargs": {}
+        },
+        "sample": {
+          "sampler": "flowmatch",
+          "sample_every": 250,
+          "width": 768,
+          "height": 768,
+          "samples": [
+            {
+              "prompt": "In the style of Sethos, This is a digital anime-style illustration featuring a young man with long, flowing black hair with green highlights and bright green eyes. He is dressed in a white and gold ornate outfit with purple accents and a yellow scarf. he is eating icecream."
+            }
+          ],
+          "neg": "",
+          "seed": 42,
+          "walk_seed": true,
+          "guidance_scale": 4,
+          "sample_steps": 25,
+          "num_frames": 1,
+          "fps": 1
+        }
+      }
+    ]
+  },
+  "meta": {
+    "name": "[name]",
+    "version": "1.0"
+  }
+}

config.yaml

@@ -0,0 +1,100 @@
+job: extension
+config:
+  name: qwen_image_sethos_lora_v1
+  process:
+  - type: ui_trainer
+    training_folder: /home/featurize/ai-toolkit/output
+    sqlite_db_path: /home/featurize/ai-toolkit/aitk_db.db
+    device: cuda
+    trigger_word: null
+    performance_log_every: 10
+    network:
+      type: lora
+      linear: 16
+      linear_alpha: 16
+      conv: 16
+      conv_alpha: 16
+      lokr_full_rank: true
+      lokr_factor: -1
+      network_kwargs:
+        ignore_if_contains: []
+    save:
+      dtype: bf16
+      save_every: 250
+      max_step_saves_to_keep: 4000
+      save_format: diffusers
+      push_to_hub: false
+    datasets:
+    - folder_path: /home/featurize/ai-toolkit/datasets/Sethos_Images_Captioned
+      control_path: null
+      mask_path: null
+      mask_min_value: 0.1
+      default_caption: ''
+      caption_ext: txt
+      caption_dropout_rate: 0.05
+      cache_latents_to_disk: false
+      is_reg: false
+      network_weight: 1
+      resolution:
+      - 512
+      - 768
+      - 1024
+      controls: []
+      shrink_video_to_frames: true
+      num_frames: 1
+      do_i2v: true
+    train:
+      batch_size: 1
+      bypass_guidance_embedding: false
+      steps: 30000
+      gradient_accumulation: 1
+      train_unet: true
+      train_text_encoder: false
+      gradient_checkpointing: true
+      noise_scheduler: flowmatch
+      optimizer: adamw8bit
+      timestep_type: weighted
+      content_or_style: balanced
+      optimizer_params:
+        weight_decay: 0.0001
+      unload_text_encoder: false
+      cache_text_embeddings: true
+      lr: 0.0001
+      ema_config:
+        use_ema: false
+        ema_decay: 0.99
+      skip_first_sample: false
+      disable_sampling: false
+      dtype: bf16
+      diff_output_preservation: false
+      diff_output_preservation_multiplier: 1
+      diff_output_preservation_class: person
+    model:
+      name_or_path: Qwen/Qwen-Image
+      quantize: true
+      qtype: uint3|ostris/accuracy_recovery_adapters/qwen_image_torchao_uint3.safetensors
+      quantize_te: true
+      qtype_te: qfloat8
+      arch: qwen_image
+      low_vram: true
+      model_kwargs: {}
+    sample:
+      sampler: flowmatch
+      sample_every: 250
+      width: 768
+      height: 768
+      samples:
+      - prompt: In the style of Sethos, This is a digital anime-style illustration
+          featuring a young man with long, flowing black hair with green highlights
+          and bright green eyes. He is dressed in a white and gold ornate outfit with
+          purple accents and a yellow scarf. he is eating icecream.
+      neg: ''
+      seed: 42
+      walk_seed: true
+      guidance_scale: 4
+      sample_steps: 25
+      num_frames: 1
+      fps: 1
+meta:
+  name: qwen_image_sethos_lora_v1
+  version: '1.0'

logs/0_log.txt

@@ -0,0 +1,205 @@
+Running 1 job
+{
+    "type": "ui_trainer",
+    "training_folder": "/home/featurize/ai-toolkit/output",
+    "sqlite_db_path": "/home/featurize/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": null,
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 16,
+        "linear_alpha": 16,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 250,
+        "max_step_saves_to_keep": 4000,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/home/featurize/ai-toolkit/datasets/Sethos_Images_Captioned",
+            "control_path": null,
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512,
+                768,
+                1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "do_i2v": true
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": true,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person"
+    },
+    "model": {
+        "name_or_path": "Qwen/Qwen-Image",
+        "quantize": true,
+        "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_torchao_uint3.safetensors",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "qwen_image",
+        "low_vram": true,
+        "model_kwargs": {}
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 768,
+        "height": 768,
+        "samples": [
+            {
+                "prompt": "In the style of Sethos, This is a digital anime-style illustration featuring a young man with long, flowing black hair with green highlights and bright green eyes. He is dressed in a white and gold ornate outfit with purple accents and a yellow scarf. he is eating icecream."
+            }
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 25,
+        "num_frames": 1,
+        "fps": 1
+    }
+}
+Using EMA
+Using SQLite database at /home/featurize/ai-toolkit/aitk_db.db
+Job ID: "5a059c5f-8a03-4ce4-9b76-67537c17317e"
+
+#############################################
+# Running job: qwen_image_sethos_lora_v1
+#############################################
+
+
+Running  1 process
+Loading Qwen Image model
+Loading transformer
+
+
+Quantizing Transformer
+Grabbing lora from the hub: ostris/accuracy_recovery_adapters/qwen_image_torchao_uint3.safetensors
+
+
+Error running job: An error occurred while downloading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 470, in http_get
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 470, in http_get
+        hf_transfer.download(hf_transfer.download(
+
+ExceptionException: : Failed too many failures in parallel (3): Request: error decoding response body (no permits available)Failed too many failures in parallel (3): Request: error decoding response body (no permits available)
+
+
+
+The above exception was the direct cause of the following exception:
+The above exception was the direct cause of the following exception:
+
+
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/home/featurize/ai-toolkit/run.py", line 120, in <module>
+  File "/home/featurize/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/home/featurize/ai-toolkit/run.py", line 108, in main
+  File "/home/featurize/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/home/featurize/ai-toolkit/run.py", line 96, in main
+  File "/home/featurize/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1534, in run
+  File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1534, in run
+        self.sd.load_model()self.sd.load_model()
+
+  File "/home/featurize/ai-toolkit/extensions_built_in/diffusion_models/qwen_image/qwen_image.py", line 102, in load_model
+  File "/home/featurize/ai-toolkit/extensions_built_in/diffusion_models/qwen_image/qwen_image.py", line 102, in load_model
+        quantize_model(self, transformer)quantize_model(self, transformer)
+
+  File "/home/featurize/ai-toolkit/toolkit/util/quantize.py", line 161, in quantize_model
+  File "/home/featurize/ai-toolkit/toolkit/util/quantize.py", line 161, in quantize_model
+        new_lora_path = hf_hub_download(new_lora_path = hf_hub_download(
+
+                                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+        return fn(*args, **kwargs)return fn(*args, **kwargs)
+
+                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1010, in hf_hub_download
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1010, in hf_hub_download
+        return _hf_hub_download_to_cache_dir(return _hf_hub_download_to_cache_dir(
+
+                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1171, in _hf_hub_download_to_cache_dir
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1171, in _hf_hub_download_to_cache_dir
+        _download_to_tmp_and_move(_download_to_tmp_and_move(
+
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1738, in _download_to_tmp_and_move
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1738, in _download_to_tmp_and_move
+        http_get(http_get(
+
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 481, in http_get
+  File "/environment/miniconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 481, in http_get
+        raise RuntimeError(raise RuntimeError(
+
+RuntimeErrorRuntimeError: : An error occurred while downloading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.An error occurred while downloading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.
+

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f0b4d2b65b581b1b331ae6ad748e713d78a87bd4de2d7bdac398635e5828112
+size 300280909

qwen_image_sethos_lora_v1_000000250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb7e4f6e072763e58b288b4bd38c69a37a8c6e4c886e3048f0cb2a429d34b599
+size 295146200

qwen_image_sethos_lora_v1_000000500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30bdd06b7ab3b3e44f79a85c94f52f77860fb8fd11cde7bbb1fc5ae77ac9d727
+size 295146200

qwen_image_sethos_lora_v1_000000750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdca99da6d92da529959965a39a1b5f95db66c025c75fadfe11b1512e28df764
+size 295146200

qwen_image_sethos_lora_v1_000001000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a074861754482bdf182da64bab6a066d99cbb30da9d8999cabe11602d49e2a38
+size 295146200

qwen_image_sethos_lora_v1_000001250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f20a322debabee31227dbc6632dca8907d92a4d817ecde3f2f905a9ab2a16c06
+size 295146200

qwen_image_sethos_lora_v1_000001500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1e21707b230417651ea5f5ee7a53b1d5b81540de3a7e6de6f89cf9923471e6d
+size 295146200

qwen_image_sethos_lora_v1_000001750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54910981c0bf177711e83b186fd69bab625194cd2bafd4365195a0eed7dd6876
+size 295146200

qwen_image_sethos_lora_v1_000002000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc6b3281a6e59550276e1ccc7ddebc56517e59b88d0efc4f6d19eadfa3b93d39
+size 295146200