Duplicate from CabalResearch/NoobAI-Flux2VAE-RectifiedFlow-0.3

Co-authored-by: Anzhc <Anzhc@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

NoobAI Flux2VAE RF Basic Workflow.json

@@ -0,0 +1,457 @@
+{
+  "id": "a4a1f988-221c-41e4-aaf1-674cec119bc1",
+  "revision": 0,
+  "last_node_id": 14,
+  "last_link_id": 15,
+  "nodes": [
+    {
+      "id": 6,
+      "type": "VAEDecodeTiled",
+      "pos": [
+        900,
+        240
+      ],
+      "size": [
+        220,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "samples",
+          "type": "LATENT",
+          "link": 2
+        },
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": 15
+        }
+      ],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "slot_index": 0,
+          "links": [
+            8
+          ]
+        }
+      ],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.26",
+        "Node name for S&R": "VAEDecodeTiled"
+      },
+      "widgets_values": [
+        512,
+        64,
+        64,
+        8
+      ],
+      "shape": 1
+    },
+    {
+      "id": 11,
+      "type": "PreviewImage",
+      "pos": [
+        1130,
+        240
+      ],
+      "size": [
+        460,
+        700
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 8
+        }
+      ],
+      "outputs": [],
+      "title": "Image Preview",
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.26",
+        "Node name for S&R": "Tuned"
+      },
+      "widgets_values": [],
+      "color": "#223",
+      "bgcolor": "#335",
+      "shape": 1
+    },
+    {
+      "id": 3,
+      "type": "KSampler",
+      "pos": [
+        650,
+        240
+      ],
+      "size": [
+        240,
+        474
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "model",
+          "type": "MODEL",
+          "link": 13
+        },
+        {
+          "name": "positive",
+          "type": "CONDITIONING",
+          "link": 10
+        },
+        {
+          "name": "negative",
+          "type": "CONDITIONING",
+          "link": 11
+        },
+        {
+          "name": "latent_image",
+          "type": "LATENT",
+          "link": 14
+        }
+      ],
+      "outputs": [
+        {
+          "name": "LATENT",
+          "type": "LATENT",
+          "slot_index": 0,
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.26",
+        "Node name for S&R": "KSampler"
+      },
+      "widgets_values": [
+        236456494,
+        "fixed",
+        24,
+        7,
+        "euler",
+        "normal",
+        1
+      ],
+      "shape": 1
+    },
+    {
+      "id": 9,
+      "type": "CLIPTextEncode",
+      "pos": [
+        10,
+        540
+      ],
+      "size": [
+        410,
+        110
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "clip",
+          "type": "CLIP",
+          "link": 6
+        }
+      ],
+      "outputs": [
+        {
+          "name": "CONDITIONING",
+          "type": "CONDITIONING",
+          "slot_index": 0,
+          "links": [
+            11
+          ]
+        }
+      ],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.26",
+        "Node name for S&R": "CLIPTextEncode"
+      },
+      "widgets_values": [
+        "worst quality, normal quality, bad anatomy"
+      ],
+      "shape": 1
+    },
+    {
+      "id": 10,
+      "type": "CLIPTextEncode",
+      "pos": [
+        10,
+        380
+      ],
+      "size": [
+        410,
+        120
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "clip",
+          "type": "CLIP",
+          "link": 5
+        }
+      ],
+      "outputs": [
+        {
+          "name": "CONDITIONING",
+          "type": "CONDITIONING",
+          "slot_index": 0,
+          "links": [
+            10
+          ]
+        }
+      ],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.26",
+        "Node name for S&R": "CLIPTextEncode"
+      },
+      "widgets_values": [
+        "masterpiece, best quality, 1girl, upper body"
+      ],
+      "shape": 1
+    },
+    {
+      "id": 1,
+      "type": "CheckpointLoaderSimple",
+      "pos": [
+        10,
+        240
+      ],
+      "size": [
+        410,
+        98
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MODEL",
+          "type": "MODEL",
+          "links": [
+            12
+          ]
+        },
+        {
+          "name": "CLIP",
+          "type": "CLIP",
+          "links": [
+            5,
+            6
+          ]
+        },
+        {
+          "name": "VAE",
+          "type": "VAE",
+          "links": [
+            15
+          ]
+        }
+      ],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.51",
+        "Node name for S&R": "CheckpointLoaderSimple"
+      },
+      "widgets_values": [
+        "NoobAI-RF-FLUX2VAE-v0.3-resumed-6e-5-000002.safetensors"
+      ],
+      "shape": 1
+    },
+    {
+      "id": 2,
+      "type": "ModelSamplingSD3",
+      "pos": [
+        430,
+        240
+      ],
+      "size": [
+        210,
+        58
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "model",
+          "type": "MODEL",
+          "link": 12
+        }
+      ],
+      "outputs": [
+        {
+          "name": "MODEL",
+          "type": "MODEL",
+          "links": [
+            13
+          ]
+        }
+      ],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.51",
+        "Node name for S&R": "ModelSamplingSD3"
+      },
+      "widgets_values": [
+        12
+      ],
+      "shape": 1
+    },
+    {
+      "id": 14,
+      "type": "EmptySDXLFlux2LatentImage",
+      "pos": [
+        430,
+        340
+      ],
+      "size": [
+        210,
+        106
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "LATENT",
+          "type": "LATENT",
+          "links": [
+            14
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "EmptySDXLFlux2LatentImage"
+      },
+      "widgets_values": [
+        832,
+        1216,
+        1
+      ],
+      "shape": 1
+    }
+  ],
+  "links": [
+    [
+      2,
+      3,
+      0,
+      6,
+      0,
+      "LATENT"
+    ],
+    [
+      5,
+      1,
+      1,
+      10,
+      0,
+      "CLIP"
+    ],
+    [
+      6,
+      1,
+      1,
+      9,
+      0,
+      "CLIP"
+    ],
+    [
+      8,
+      6,
+      0,
+      11,
+      0,
+      "IMAGE"
+    ],
+    [
+      10,
+      10,
+      0,
+      3,
+      1,
+      "CONDITIONING"
+    ],
+    [
+      11,
+      9,
+      0,
+      3,
+      2,
+      "CONDITIONING"
+    ],
+    [
+      12,
+      1,
+      0,
+      2,
+      0,
+      "MODEL"
+    ],
+    [
+      13,
+      2,
+      0,
+      3,
+      0,
+      "MODEL"
+    ],
+    [
+      14,
+      14,
+      0,
+      3,
+      3,
+      "LATENT"
+    ],
+    [
+      15,
+      1,
+      2,
+      6,
+      1,
+      "VAE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.9229599817706534,
+      "offset": [
+        541.4328725112438,
+        286.4892025504186
+      ]
+    },
+    "frontendVersion": "1.32.9",
+    "VHS_latentpreview": false,
+    "VHS_latentpreviewrate": 0,
+    "VHS_MetadataImage": true,
+    "VHS_KeepIntermediate": true,
+    "workflowRendererVersion": "LG"
+  },
+  "version": 0.4
+}

NoobAI Flux2VAE RF v0.3 - Aesthetic.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b40460acab7c8dc79eb9ef7c3ea41a3886c92c390e39bb744cf542aa33a9829c
+size 6939151202

NoobAI Flux2VAE RF v0.3 - Base.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00b3c576917dfc8bf38743179311761533c33097593203a3713cde973ccb02e2
+size 6939151234

README.md

@@ -0,0 +1,259 @@
+---
+license: other
+license_name: fair-ai-public-license-1.0-sd
+license_link: https://freedevproject.org/faipl-1.0-sd/
+base_model:
+- CabalResearch/NoobAI-Flux2VAE-RectifiedFlow
+library_name: diffusers
+---
+## Model Details
+
+A continuation of the [NoobAI Flux2 VAE experiment](https://huggingface.co/CabalResearch/NoobAI-Flux2VAE-RectifiedFlow)
+
+More info on supporting us: [click me](https://huggingface.co/CabalResearch/NoobAI-Flux2VAE-RectifiedFlow-0.3#potential-future) 
+
+### Model Description
+
+Resumed for 4 more epochs, model has shown a nice improvement. We observe good convergence to new details, that were hard to achieve on prior arch. Compositions and stability are strongly improved relative to Epoch 2, as well as downstream trainability (like LoRAs).
+
+Current state is usable for normal generations, so we encourage you to try it. We will provide an [easy node for ComfyUI](https://github.com/Anzhc/SDXL-Flux2VAE-ComfyUI-Node), as well as basic workflow. If you are an A1111 user, please use [ReForge](https://github.com/Panchovix/stable-diffusion-webui-reForge), it has native support, instructions will be below.
+
+
+![e2-6 progress](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/luRNeWtDiT78z-cfuMI8k.png)
+
+Once again, we are working with limited compute, but are quite happy with the result so far, and hope to continue working on the model.
+
+
+- **Developed by:** Cabal Research (Bluvoll, Anzhc)
+- **Funded by:** Community
+- **License:** [fair-ai-public-license-1.0-sd](https://freedevproject.org/faipl-1.0-sd/)
+- **Finetuned from model:** [NoobAI Flux2 VAE experiment](https://huggingface.co/CabalResearch/NoobAI-Flux2VAE-RectifiedFlow)
+
+
+## Bias and Limitations
+
+While we are seeing new level of details, it is still early to call it a day. Complex intersections, extremely small details, abstract and wide shots will pose a significant challenge to model and result in noise-like patterns, but we see steady progression in resolving that noise through epochs.
+
+Most biases of official dataset will apply(Blue Archive, etc.).
+
+We are yet to get to steady composition and anatomy, but good LoRAs help drastically with this at current stage.
+
+
+## Model Output Examples
+
+![00182-1507939795](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/aGUOjzYm857vyJ4Bw3A3s.png)
+
+![00175-1757655322](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/910YmdSXDkPfixIwyKbfv.png)
+
+![00180-26729011](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/19dUzddyfZDdBfWI38B5P.png)
+
+![00177-246779580](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/6ZZQd3m5TeG2OQYbaDSeq.png)
+
+![00178-1560785548](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/RzsGySlS8YEBLginowFSZ.png)
+
+![00181-1519334009](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/2dZug4PjLhmgr7YSTaOFe.png)
+
+![00179-1426592070](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/gsWqweJ-__V2L4-a0sncF.png)
+
+![00184-3857447066](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/4ds-97--ZEB-BA5w1EzI7.png)
+
+![00187-2418047636](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/5Os3cK3mwg5YntqDrhSoF.png)
+
+![00186-1104627563](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/8swj56l_Pd5K5Mps9oyUn.png)
+
+![00188-3362709](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/ANSwUK8_k-m1uCGMVUSB9.png)
+
+P.S. We are pretty bad at generating images, on Epoch 2 we've seen quite a few examples of much better generations that what we've shown, wonder if this time it will also be the case.
+
+# Recommendations
+
+### Inference
+
+#### Comfy
+
+![image](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/urIE5XGaUpjkaXutaiebs.png)
+
+(Workflow is available alongside model in repo)
+We will provide a Node, and hope it will be adapted natively in main repo eventually:  
+**https://github.com/Anzhc/SDXL-Flux2VAE-ComfyUI-Node**
+
+Seems like you don't need to use the node itself, patch is applied without it.
+
+
+Apparently works in [SwarmUI](https://github.com/mcmonkeyprojects/SwarmUI) as is.
+
+Same as your normal inference, but with addition of SD3 sampling node, as this model is Flow-based.
+
+Recommended Parameters:  
+**Sampler**: Euler, Euler A, DPM++ SDE, etc.  
+**Steps**: 20-28  
+**CFG**: 6-9  
+**Shift**: 3-12  
+**Schedule**: Normal/Simple/SGM Uniform/Quadratic  
+**Positive Quality Tags**: `masterpiece, best quality`  
+**Negative Tags**: `worst quality, normal quality, bad anatomy`  
+
+
+#### A1111 WebUI
+
+(All screenshots are repeating our RF release, as there is no difference in setup)
+
+
+Recommended WebUI: [ReForge](https://github.com/Panchovix/stable-diffusion-webui-reForge) - has native support for Flow models, and we've PR'd our native support for Flux2vae-based SDXL modification.
+
+
+**How to use in ReForge**:
+
+![изображение](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/UV5Yp66H7YlccdQqborPf.png)
+(ignore Sigma max field at the top, this is not used in RF)
+
+Support for RF in ReForge is being implemented through a built-in extension:
+
+![изображение](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/LpMF0lmC96X001Au9fFU_.png)
+
+
+![imagen](https://cdn-uploads.huggingface.co/production/uploads/634cf025fe861cc73a2e7dd0/siEBfX41E6D_DUEqaXMgs.png)
+
+Set parameters to that, and you're good to go.
+
+Flux2VAE does not currently have an appropriate high quality preview method, please use Approx Cheap option, which would allow you to see simple PCA projection(ReForge).
+
+Recommended Parameters:  
+**Sampler**: Euler A Comfy RF, Euler A2, Euler, DPM++ SDE Comfy, etc. **ALL VARIANTS MUST BE RF OR COMFY, IF AVAILABLE. In ComfyUI routing is automatic, but not in the case of WebUI.**  
+**Steps**: 20-28  
+**CFG**: 6-9  
+**Shift**: 3-12  
+**Schedule**: Normal/Simple/SGM Uniform  
+**Positive Quality Tags**: `masterpiece, best quality`  
+**Negative Tags**: `worst quality, normal quality, bad anatomy`  
+
+**ADETAILER FIX FOR RF**:
+By default, Adetailer discards Advanced Model Sampling extension, which breaks RF. You need to add AMS to this part of settings:
+
+![изображение](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/RQMtfm5Xi3V7oNsqXoZJN.png)
+
+Add: `advanced_model_sampling_script,advanced_model_sampling_script_backported` to there.
+
+If that does not work, go into adetailer extension, find args.py, open it, replace _builtin_scripts like this:
+
+![изображение](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/rmnS-i_kciJzTZmeR-mGP.png)
+
+Here is a copypaste for easy copy:
+```
+_builtin_script = (
+    "advanced_model_sampling_script",
+    "advanced_model_sampling_script_backported",
+    "hypertile_script",
+    "soft_inpainting",
+)
+```
+
+Or use my fork of Adetailer - https://github.com/Anzhc/aadetailer-reforge
+
+## Training
+
+### Model Composition
+(Relative to base it's trained from)
+
+Unet: Same
+CLIP L: Same, Frozen
+CLIP G: Same, Frozen
+VAE: [Flux2 VAE](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main/vae)
+
+
+### Training Details
+(Main Stage Training)
+
+**Samples seen**(unbatched steps): ~50 million samples seen  
+**Learning Rate**: 6e-5 (General Training) and 3e-5 (Aesthetic)  
+**Effective Batch size**: ~1400 (86x8 Batch Size, Accumulation 2 )
+**Precision**: Mixed BF16  
+**Optimizer**: AdamW8bit with Kahan Summation  
+**Weight Decay**: 0.01  
+**Schedule**: Constant with warmup  
+**Timestep Sampling Strategy**: Logit-Normal -0.2 1.5 (sometimes referred to as Lognorm), Shift 2.5  
+**Text Encoders**: Frozen  
+**Keep Token**: False  
+**Tag Dropout**: 10%  
+**Uncond Dropout**: 10%  
+**Shuffle**: True  
+
+**VAE Conv Padding**: False  
+**VAE Shift**: 0.0760  
+**VAE Scale**: 0.6043
+
+**Additional Features used**: Protected Tags, Cosine Optimal Transport.
+
+#### Training Data
+
+6 epochs of the original NoobAI dataset, including images up to October 2024, minus screencap data(was not shared).
+
+
+### LoRA Training
+
+Current state of the model provides adequate trainability, but expect the need to train for a bit longer, as we are still undertrained.
+My current style training settings (Anzhc):
+
+**Learning Rate**: tested up to **7.5e-4**  
+**Batch Size**: 144 (6 real * 24 accum), using SGA(Stochastic Gradient Accumulation) - without SGA I probably would lower accum to 4-8.  
+**Optimizer**: Adamw8bit with Kahan summation  
+**Schedule**: ReREX (Use REX for simplicity, or Cosine annealing)  
+**Precision**: Full BF16  
+**Weight Decay**: 0.02  
+**Timestep Sampling Strategy**: Logit-Normal(either 0.0 1.0, or -0.2 1.5), Shift 2.5-4.5  
+
+**Dim/Alpha/Conv/Alpha**: 24/24/24/24 (Lycoris/Locon)  
+
+**Text Encoders**: Frozen  
+
+**Optimal Transport**: True  
+
+**Expected Dataset Size**: 100-200 images (Can be even 10, but balance with repeats to roughly this target.)  
+**Epochs**: 50  
+
+Concepts seem to train at similar speed to prior NoobAI models, but have not tested explicitly.
+
+### Hardware
+
+Model was trained on cloud 8xH200 node.
+
+### Software
+
+Custom fork of [SD-Scripts](https://github.com/bluvoll/sd-scripts)(maintained by Bluvoll)
+
+## Acknowledgements
+
+### Special Thanks
+
+**To a special supporter who singlehandidly sponsored whole run and preferred to stay anonymous**
+
+**Additional donators**
+-mfcg
+-holo
+-dyshidrosis
+-remix
+-edf
+
+---
+
+# Support
+If you wish to support our continuous effort of making waifus 0.2% better, you can do it here:  
+
+### **https://ko-fi.com/bluvoll** (Blu, donate here to support training)
+
+https://ko-fi.com/anzhc (Anzhc, non-training, just survival)
+
+![image](https://cdn-uploads.huggingface.co/production/uploads/633b43d29fe04b13f46c8988/qpcOFLCI-B7QBADEYwMhA.png)
+
+BTC:  `37fLcfxX5ewhJXnb3T9Qzu9jiSLjVtoUJX`  
+ETH:  `0xfdF54655796bf2F5bf75192AeB562F8656c1C39E`
+
+Send DM to Blu if you want to donate on another network.  
+
+# Potential future
+
+**Expected Compute Needed**: We still consider full run to be in range of 20+ epochs, but no longer think that it is the bare minimum for stable model, as progress with just current 6 epochs has been quite drastic in that regard. 10 epochs are likely a good marker for that.  
+
+**Dataset**: We would love to start processing of the booru data with our in-house classification models to fix some of the glaring issues with the default Danbooru dataset, as well as thorough processing to some of the concepts, but as of now we don't have budget to rent a dedicated server for persistent storage.
+
+**Future Training**: We have confirmation from Sponsor that we would continue training of the model beyond Epoch 6, but it will resume after a short break.