feat: upload LoRA v2 weights (trained on quantized hf_snapped dataset)

config.yaml

@@ -1,130 +1,36 @@
 model:
   pretrained_model_name: black-forest-labs/FLUX.2-klein-4B
   dtype: bfloat16
-  enable_cpu_offload: true
 lora:
   rank: 64
   alpha: 128
   dropout: 0.0
   target_modules:
-  - transformer_blocks.0.attn.to_q
-  - transformer_blocks.0.attn.to_k
-  - transformer_blocks.0.attn.to_v
-  - transformer_blocks.0.attn.to_out.0
-  - transformer_blocks.0.attn.add_q_proj
-  - transformer_blocks.0.attn.add_k_proj
-  - transformer_blocks.0.attn.add_v_proj
-  - transformer_blocks.0.attn.to_add_out
-  - transformer_blocks.0.ff.linear_in
-  - transformer_blocks.0.ff.linear_out
-  - transformer_blocks.0.ff_context.linear_in
-  - transformer_blocks.0.ff_context.linear_out
-  - transformer_blocks.1.attn.to_q
-  - transformer_blocks.1.attn.to_k
-  - transformer_blocks.1.attn.to_v
-  - transformer_blocks.1.attn.to_out.0
-  - transformer_blocks.1.attn.add_q_proj
-  - transformer_blocks.1.attn.add_k_proj
-  - transformer_blocks.1.attn.add_v_proj
-  - transformer_blocks.1.attn.to_add_out
-  - transformer_blocks.1.ff.linear_in
-  - transformer_blocks.1.ff.linear_out
-  - transformer_blocks.1.ff_context.linear_in
-  - transformer_blocks.1.ff_context.linear_out
-  - transformer_blocks.2.attn.to_q
-  - transformer_blocks.2.attn.to_k
-  - transformer_blocks.2.attn.to_v
-  - transformer_blocks.2.attn.to_out.0
-  - transformer_blocks.2.attn.add_q_proj
-  - transformer_blocks.2.attn.add_k_proj
-  - transformer_blocks.2.attn.add_v_proj
-  - transformer_blocks.2.attn.to_add_out
-  - transformer_blocks.2.ff.linear_in
-  - transformer_blocks.2.ff.linear_out
-  - transformer_blocks.2.ff_context.linear_in
-  - transformer_blocks.2.ff_context.linear_out
-  - transformer_blocks.3.attn.to_q
-  - transformer_blocks.3.attn.to_k
-  - transformer_blocks.3.attn.to_v
-  - transformer_blocks.3.attn.to_out.0
-  - transformer_blocks.3.attn.add_q_proj
-  - transformer_blocks.3.attn.add_k_proj
-  - transformer_blocks.3.attn.add_v_proj
-  - transformer_blocks.3.attn.to_add_out
-  - transformer_blocks.3.ff.linear_in
-  - transformer_blocks.3.ff.linear_out
-  - transformer_blocks.3.ff_context.linear_in
-  - transformer_blocks.3.ff_context.linear_out
-  - transformer_blocks.4.attn.to_q
-  - transformer_blocks.4.attn.to_k
-  - transformer_blocks.4.attn.to_v
-  - transformer_blocks.4.attn.to_out.0
-  - transformer_blocks.4.attn.add_q_proj
-  - transformer_blocks.4.attn.add_k_proj
-  - transformer_blocks.4.attn.add_v_proj
-  - transformer_blocks.4.attn.to_add_out
-  - transformer_blocks.4.ff.linear_in
-  - transformer_blocks.4.ff.linear_out
-  - transformer_blocks.4.ff_context.linear_in
-  - transformer_blocks.4.ff_context.linear_out
-  - single_transformer_blocks.0.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.0.attn.to_out
-  - single_transformer_blocks.1.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.1.attn.to_out
-  - single_transformer_blocks.2.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.2.attn.to_out
-  - single_transformer_blocks.3.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.3.attn.to_out
-  - single_transformer_blocks.4.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.4.attn.to_out
-  - single_transformer_blocks.5.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.5.attn.to_out
-  - single_transformer_blocks.6.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.6.attn.to_out
-  - single_transformer_blocks.7.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.7.attn.to_out
-  - single_transformer_blocks.8.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.8.attn.to_out
-  - single_transformer_blocks.9.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.9.attn.to_out
-  - single_transformer_blocks.10.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.10.attn.to_out
-  - single_transformer_blocks.11.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.11.attn.to_out
-  - single_transformer_blocks.12.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.12.attn.to_out
-  - single_transformer_blocks.13.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.13.attn.to_out
-  - single_transformer_blocks.14.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.14.attn.to_out
-  - single_transformer_blocks.15.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.15.attn.to_out
-  - single_transformer_blocks.16.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.16.attn.to_out
-  - single_transformer_blocks.17.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.17.attn.to_out
-  - single_transformer_blocks.18.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.18.attn.to_out
-  - single_transformer_blocks.19.attn.to_qkv_mlp_proj
-  - single_transformer_blocks.19.attn.to_out
-  - time_guidance_embed.timestep_embedder.linear_1
-  - time_guidance_embed.timestep_embedder.linear_2
-  - double_stream_modulation_img.linear
-  - double_stream_modulation_txt.linear
-  - x_embedder
+  - to_q
+  - to_k
+  - to_v
+  - to_out.0
+  - add_q_proj
+  - add_k_proj
+  - add_v_proj
+  - to_add_out
+  - linear_in
+  - linear_out
+  - to_qkv_mlp_proj
+  - proj_out
   - context_embedder
+  - x_embedder
   - single_stream_modulation.linear
-  - norm_out.linear
-  - proj_out
+  - double_stream_modulation_img.linear
+  - double_stream_modulation_txt.linear
   use_rslora: true
 dataset:
-  data_dir: ../training_data/synthetic/images
+  data_dir: /home/gero/GitHub/DeepLearning_Lab/SpriteForge/training_data/hf_snapped
   caption_ext: txt
   resolution: 512
   center_crop: false
   random_flip: true
-output_dir: ./output/sprite_lora_v1_continued
-resume_from_checkpoint: ./output/sprite_lora_v1_continued/step_500
+output_dir: ./output/pixel-art-lora-v2
 num_train_steps: 1000
 batch_size: 1
 gradient_accumulation_steps: 4
@@ -132,6 +38,8 @@ learning_rate: 0.0001
 lr_scheduler: cosine_with_restarts
 warmup_steps: 100
 max_grad_norm: 1.0
+gradient_checkpointing: true
+offload_frozen_to_cpu: true
 optimizer: adamw_8bit
 beta1: 0.9
 beta2: 0.999
@@ -147,4 +55,5 @@ sample_guidance_scale: 1.0
 push_to_hub: false
 hub_model_id: null
 hub_private: false
+save_comfyui_format: true
 trigger_word: pixel art sprite

pytorch_lora_weights.comfyui.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24e938f510f5dd0c890ac8b1078f4abb87a50c9c053b2e85c44821c0f30011ad
+size 325276656

pytorch_lora_weights.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:499c431145654822311b9657613d135068046901f8b8ef107021a989bbc1add4
-size 307550792
+oid sha256:d0414baac1e19fe28c00dffb1c09eaffe2dbaf7928821d6d53d5ebd3ab903db8
+size 325279584