Duplicate from GD-ML/FLUX-Text

Co-authored-by: lan <rrui@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,59 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/comfyui.png filter=lfs diff=lfs merge=lfs -text
+assets/comfyui2.png filter=lfs diff=lfs merge=lfs -text
+assets/flux-text.png filter=lfs diff=lfs merge=lfs -text
+assets/gradio_1.png filter=lfs diff=lfs merge=lfs -text
+assets/gradio_2.png filter=lfs diff=lfs merge=lfs -text
+assets/method.png filter=lfs diff=lfs merge=lfs -text
+assets/method_result.png filter=lfs diff=lfs merge=lfs -text
+assets/new_img1.png filter=lfs diff=lfs merge=lfs -text
+assets/new_img2.png filter=lfs diff=lfs merge=lfs -text
+assets/ori_img1.png filter=lfs diff=lfs merge=lfs -text
+assets/ori_img2.png filter=lfs diff=lfs merge=lfs -text
+assets/video1.gif filter=lfs diff=lfs merge=lfs -text
+assets/video2.gif filter=lfs diff=lfs merge=lfs -text
+assets/video_end1.png filter=lfs diff=lfs merge=lfs -text
+assets/video_end2.png filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/LSVT/data-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/MTWI2018/data-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/ReCTS/data-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/laion/data_v1.1-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/wukong_1of5/data_v1.1-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/wukong_2of5/data_v1.1-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/wukong_3of5/data_v1.1-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/wukong_4of5/data_v1.1-info.json filter=lfs diff=lfs merge=lfs -text
+data_text_recog_glyph/wukong_5of5/data_v1.1-info.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,341 @@
+---
+license: mit
+pipeline_tag: image-to-image
+library_name: diffusers
+tags:
+- text-editing
+- multilingual
+- diffusion-transformer
+- diffusion-model
+---
+
+# FLUX-Text: A Simple and Advanced Diffusion Transformer Baseline for Scene Text Editing
+
+<a href='https://amap-ml.github.io/FLUX-text/'><img src='https://img.shields.io/badge/Project-Page-green'></a>
+<a href='https://arxiv.org/abs/2505.03329'><img src='https://img.shields.io/badge/Technique-Report-red'></a> 
+<a href="https://github.com/AMAP-ML/FluxText"><img src="https://img.shields.io/badge/GitHub-Code-blue.svg?logo=github&"></a>
+<a href="https://huggingface.co/GD-ML/FLUX-Text/"><img src="https://img.shields.io/badge/%F0%9F%A4%97_HuggingFace-Model-ffbd45.svg" alt="HuggingFace"></a>
+
+> *[Rui Lan](https://scholar.google.com/citations?user=zwVlWXwAAAAJ&hl=zh-CN), [Yancheng Bai](https://scholar.google.com/citations?hl=zh-CN&user=Ilx8WNkAAAAJ&view_op=list_works&sortby=pubdate), [Xu Duan](https://scholar.google.com/citations?hl=zh-CN&user=EEUiFbwAAAAJ), [Mingxing Li](https://scholar.google.com/citations?hl=zh-CN&user=-pfkprkAAAAJ), [Lei Sun](https://allylei.github.io), [Xiangxiang Chu](https://scholar.google.com/citations?hl=zh-CN&user=jn21pUsAAAAJ&view_op=list_works&sortby=pubdate)*
+> <br>
+> ALibaba Group
+
+<img src='assets/flux-text.png'>
+
+## 📖 Overview
+* **Motivation:** Scene text editing is a challenging task that aims to modify or add text in images while maintaining the fidelity of newly generated text and visual coherence with the background. The main challenge of this task is that we need to edit multiple line texts with diverse language attributes (e.g., fonts, sizes, and styles), language types (e.g., English, Chinese), and visual scenarios (e.g., poster, advertising, gaming).
+* **Contribution:** We propose FLUX-Text, a novel text editing framework for editing multi-line texts in complex visual scenes. By incorporating a lightweight Condition Injection LoRA module, Regional text perceptual loss, and two-stage training strategy, we significantly significant improvements on both Chinese and English benchmarks.
+<img src='assets/method.png'>
+
+## News
+
+- **2025-07-16**: 🔥 Update comfyui node. We have decoupled the FLUX-Text node to support the use of more basic nodes. Due to differences in node computation in ComfyUI, if you need more consistent results, you should set min_length to 512 in the [code](https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/text_encoders/flux.py#L12).
+
+<div align="center">
+<table>
+<tr>
+    <td><img src="assets/comfyui2.png" alt="workflow/FLUX-Text-Basic-Workflow.json" width="400"/></td>
+</tr>
+<tr>
+    <td align="center">workflow/FLUX-Text-Basic-Workflow.json</td>
+</tr>
+</table>
+</div>
+
+- **2025-07-13**: 🔥 The training code has been updated. The code now supports multi-scale training.
+
+- **2025-07-13**: 🔥 Update the low-VRAM version of the Gradio demo, which It currently requires 25GB of VRAM to run. Looking forward to more efficient, lower-memory solutions from the community.
+
+- **2025-07-08**: 🔥 ComfyUI Node is supported! You can now build an workflow based on FLUX-Text for editing posters. It is definitely worth trying to set up a workflow to automatically enhance product image service information and service scope. Meanwhile, utilizing the first and last frames enables the creation of video data with text effects. Thanks to the [community work](https://github.com/AMAP-ML/FluxText/issues/4), FLUX-Text was run on 8GB VRAM. 
+
+<div align="center">
+<table>
+<tr>
+    <td><img src="assets/comfyui.png" alt="workflow/FLUX-Text-Workflow.json" width="400"/></td>
+</tr>
+<tr>
+    <td align="center">workflow/FLUX-Text-Workflow.json</td>
+</tr>
+</table>
+</div>
+
+<div align="center">
+<table>
+<tr>
+    <td><img src="assets/ori_img1.png" alt="assets/ori_img1.png" width="200"/></td>
+    <td><img src="assets/new_img1.png" alt="assets/new_img1.png" width="200"/></td>
+    <td><img src="assets/ori_img2.png" alt="assets/ori_img2.png" width="200"/></td>
+    <td><img src="assets/new_img2.png" alt="assets/new_img2.png" width="200"/></td>
+</tr>
+<tr>
+    <td align="center">original image</td>
+    <td align="center">edited image</td>
+    <td align="center">original image</td>
+    <td align="center">edited image</td>
+</tr>
+</table>
+</div>
+
+<div align="center">
+<table>
+<tr>
+    <td><img src="assets/video_end1.png" alt="assets/video_end1.png" width="400"/></td>
+    <td><img src="assets/video1.gif" alt="assets/video1.gif" width="400"/></td>
+</tr>
+<tr>
+    <td><img src="assets/video_end2.png" alt="assets/video_end2.png" width="400"/></td>
+    <td><img src="assets/video2.gif" alt="assets/video2.gif" width="400"/></td>
+</tr>
+<tr>
+    <td align="center">last frame</td>
+    <td align="center">video</td>
+</tr>
+</table>
+</div>
+
+- **2025-07-04**: 🔥 We have released gradio demo! You can now try out FLUX-Text.
+
+<div align="center">
+<table>
+<tr>
+    <td><img src="assets/gradio_1.png" alt="Example 1" width="400"/></td>
+    <td><img src="assets/gradio_2.png" alt="Example 2" width="400"/></td>
+</tr>
+<tr>
+    <td align="center">Example 1</td>
+    <td align="center">Example 2</td>
+</tr>
+</table>
+</div>
+
+- **2025-07-03**: 🔥 We have released our [pre-trained checkpoints](https://huggingface.co/GD-ML/FLUX-Text/) on Hugging Face! You can now try out FLUX-Text with the official weights.
+
+- **2025-06-26**: ⭐️ Inference and evaluate code are released. Once we have ensured that everything is functioning correctly, the new model will be merged into this repository.
+
+## Todo List
+1. - [x] Inference code 
+2. - [x] Pre-trained weights 
+3. - [x] Gradio demo
+4. - [x] ComfyUI
+5. - [x] Training code
+
+## 🛠️ Installation
+
+We recommend using Python 3.10 and PyTorch with CUDA support. To set up the environment:
+
+```bash
+# Create a new conda environment
+conda create -n flux_text python=3.10
+conda activate flux_text
+
+# Install other dependencies
+pip install -r requirements.txt
+pip install flash_attn --no-build-isolation
+pip install Pillow==9.5.0
+```
+
+## 🤗 Model Introduction
+
+FLUX-Text is an open-source version of the scene text editing model. FLUX-Text can be used for editing posters, emotions, and more. The table below displays the list of text editing models we currently offer, along with their foundational information.
+
+<table style="border-collapse: collapse; width: 100%;">
+  <tr>
+    <th style="text-align: center;">Model Name</th>
+    <th style="text-align: center;">Image Resolution</th>
+    <th style="text-align: center;">Memory Usage</th>
+    <th style="text-align: center;">English Sen.Acc</th>
+    <th style="text-align: center;">Chinese Sen.Acc</th>
+    <th style="text-align: center;">Download Link</th>
+  </tr>
+  <tr>
+    <th style="text-align: center;">FLUX-Text-512</th>
+    <th style="text-align: center;">512*512</th>
+    <th style="text-align: center;">34G</th>
+    <th style="text-align: center;">0.8419</th>
+    <th style="text-align: center;">0.7132</th>
+    <th style="text-align: center;"><a href="https://huggingface.co/GD-ML/FLUX-Text/tree/main/model_512">🤗 HuggingFace</a></th>
+  </tr>
+  <tr>
+    <th style="text-align: center;">FLUX-Text</th>
+    <th style="text-align: center;">Multi Resolution</th>
+    <th style="text-align: center;">34G for (512*512)</th>
+    <th style="text-align: center;">0.8228</th>
+    <th style="text-align: center;">0.7161</th>
+    <th style="text-align: center;"><a href="https://huggingface.co/GD-ML/FLUX-Text/tree/main/model_multisize">🤗 HuggingFace</a></th>
+  </tr>
+</table>
+
+## 🔥 ComfyUI
+
+<details close>
+<summary> Installing via GitHub </summary>
+
+First, install and set up [ComfyUI](https://github.com/comfyanonymous/ComfyUI), and then follow these steps:
+
+1. **Clone FLUXText Repository**:
+   ```shell
+   git clone https://github.com/AMAP-ML/FluxText.git
+   ```
+
+2. **Install FluxText**:
+   ```shell
+   cd FluxText && pip install -r requirements.txt
+   ```
+
+3. **Integrate FluxText Comfy Nodes with ComfyUI**:
+   - **Symbolic Link (Recommended)**:
+     ```shell
+     ln -s $(pwd)/ComfyUI-fluxtext path/to/ComfyUI/custom_nodes/
+     ```
+   - **Copy Directory**:
+     ```shell
+     cp -r ComfyUI-fluxtext path/to/ComfyUI/custom_nodes/
+     ```
+
+</details>
+
+## 🔥 Quick Start
+
+Here's a basic example of using FLUX-Text:
+
+```python
+import numpy as np
+from PIL import Image
+import torch
+import yaml
+
+from src.flux.condition import Condition
+from src.flux.generate_fill import generate_fill
+from src.train.model import OminiModelFIll
+from safetensors.torch import load_file
+
+config_path = ""
+lora_path = ""
+with open(config_path, "r") as f:
+    config = yaml.safe_load(f)
+model = OminiModelFIll(
+        flux_pipe_id=config["flux_path"],
+        lora_config=config["train"]["lora_config"],
+        device=f"cuda",
+        dtype=getattr(torch, config["dtype"]),
+        optimizer_config=config["train"]["optimizer"],
+        model_config=config.get("model", {}),
+        gradient_checkpointing=True,
+        byt5_encoder_config=None,
+    )
+
+state_dict = load_file(lora_path)
+state_dict_new = {x.replace('lora_A', 'lora_A.default').replace('lora_B', 'lora_B.default').replace('transformer.', ''): v for x, v in state_dict.items()}
+model.transformer.load_state_dict(state_dict_new, strict=False)
+pipe = model.flux_pipe
+
+prompt = "lepto college of education, the written materials on the picture: LESOTHO , COLLEGE OF , RE BONA LESELI LESEL , EDUCATION ."
+hint = Image.open("assets/hint.png").resize((512, 512)).convert('RGB')
+img = Image.open("assets/hint_imgs.jpg").resize((512, 512))
+condition_img = Image.open("assets/hint_imgs_word.png").resize((512, 512)).convert('RGB')
+hint = np.array(hint) / 255
+condition_img = np.array(condition_img)
+condition_img = (255 - condition_img) / 255
+condition_img = [condition_img, hint, img]
+position_delta = [0, 0]
+condition = Condition(
+                condition_type='word_fill',
+                condition=condition_img,
+                position_delta=position_delta,
+            )
+generator = torch.Generator(device="cuda")
+res = generate_fill(
+    pipe,
+    prompt=prompt,
+    conditions=[condition],
+    height=512,
+    width=512,
+    generator=generator,
+    model_config=config.get("model", {}),
+    default_lora=True,
+)
+res.images[0].save('flux_fill.png')
+```
+
+## 🤗 gradio
+
+You can upload the glyph image and mask image to edit text region. Or you can use `manual edit` to obtain glyph image and mask image.
+
+first, download the model weight and config in [HuggingFace](https://huggingface.co/GD-ML/FLUX-Text)
+
+```bash
+python app.py --model_path xx.safetensors --config_path config.yaml
+```
+
+## 💪🏻  Training
+
+1. Download training dataset [**AnyWord-3M**](https://modelscope.cn/datasets/iic/AnyWord-3M/summary) from ModelScope, unzip all \*.zip files in each subfolder, then open *\*.json* and modify the `data_root` with your own path of *imgs* folder for each sub dataset.
+
+2. Replace the old annotations in AnyWord with the new [annotations](https://huggingface.co/GD-ML/FLUX-Text/tree/main/data_text_recog_glyph). Change the dataset annotations path and image_root in [src/train/data_word.py](https://github.com/AMAP-ML/FluxText/blob/main/src/train/data_word.py#L538).
+
+```python
+json_paths = [
+        ['dataset/Anyword/data_text_recog_glyph/Art/data-info.json', 'AnyWord-3M/ocr_data/Art/imgs/'],
+        ['dataset/Anyword/data_text_recog_glyph/COCO_Text/data-info.json', 'AnyWord-3M/ocr_data/COCO_Text/imgs/'],
+        ['dataset/Anyword/data_text_recog_glyph/icdar2017rctw/data-info.json', 'AnyWord-3M/ocr_data/icdar2017rctw/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/LSVT/data-info.json', 'AnyWord-3M/ocr_data/LSVT/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/mlt2019/data-info.json', 'AnyWord-3M/ocr_data/mlt2019/imgs/'],
+        ['dataset/Anyword/data_text_recog_glyph/MTWI2018/data-info.json', 'AnyWord-3M/ocr_data/MTWI2018/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/ReCTS/data-info.json', 'AnyWord-3M/ocr_data/ReCTS/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/laion/data_v1.1-info.json', 'AnyWord-3M/laion/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/wukong_1of5/data_v1.1-info.json', 'AnyWord-3M/wukong_1of5/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/wukong_2of5/data_v1.1-info.json', 'AnyWord-3M/wukong_2of5/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/wukong_3of5/data_v1.1-info.json', 'AnyWord-3M/wukong_3of5/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/wukong_4of5/data_v1.1-info.json', 'AnyWord-3M/wukong_4of5/imgs'],
+        ['dataset/Anyword/data_text_recog_glyph/wukong_5of5/data_v1.1-info.json', 'AnyWord-3M/wukong_5of5/imgs'],
+        ]
+```
+
+3. Download the ODM weights in [HuggingFace](https://huggingface.co/GD-ML/FLUX-Text/blob/main/epoch_100.pt) and change `odm_loss/modelpath` in the [config file](https://github.com/AMAP-ML/FluxText/blob/main/train/config/word_multi_size.yaml#L60).
+
+3. (Optional) Download the pretrained weight in [HuggingFace](https://huggingface.co/GD-ML/FLUX-Text) and change `reuse_lora_path` in the [config file](https://github.com/AMAP-ML/FluxText/blob/main/train/config/word_multi_size.yaml#L44).
+
+4. Run the training scripts. With 48GB of VRAM, you can train at 512×512 resolution with a batch size of 2 in LoRA rank 8.
+
+```bash
+bash train/script/train_word.sh
+```
+
+
+## 📊 Evaluation
+
+For [Anytext-benchmark](https://modelscope.cn/datasets/iic/AnyText-benchmark/summary), please set the **config_path**, **model_path**, **json_path**, **output_dir** in the `eval/gen_imgs_anytext.sh` and generate the text editing results.
+
+```bash
+bash eval/gen_imgs_anytext.sh
+```
+
+For `Sen.ACC, NED, FID and LPIPS` evaluation, use the scripts in the `eval` folder.
+
+```bash
+bash eval/eval_ocr.sh
+bash eval/eval_fid.sh
+bash eval/eval_lpips.sh
+```
+
+## 📈 Results
+
+<img src='assets/method_result.png'>
+
+## 🌹 Acknowledgement
+
+Our work is primarily based on [OminiControl](https://github.com/Yuanshi9815/OminiControl), [AnyText](https://github.com/tyxsspa/AnyText), [Open-Sora](https://github.com/hpcaitech/Open-Sora), [Phantom](https://github.com/Phantom-video/Phantom). We are sincerely grateful for their excellent works.
+
+## 📚 Citation
+
+If you find our paper and code helpful for your research, please consider starring our repository ⭐ and citing our work ✏️.
+```bibtex
+@misc{lan2025fluxtext,
+    title={FLUX-Text: A Simple and Advanced Diffusion Transformer Baseline for Scene Text Editing},
+    author={Rui Lan and Yancheng Bai and Xu Duan and Mingxing Li and Lei Sun and Xiangxiang Chu},
+    year={2025},
+    eprint={2505.03329},
+    archivePrefix={arXiv},
+    primaryClass={cs.CV}
+}
+```

data_text_recog_glyph/LSVT/data-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7726b7363dab7fb21115f783d65a8bfe07a8c8b76baa24ec610829206fc51228
+size 20801296

data_text_recog_glyph/MTWI2018/data-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3e881d778c748958f36436b23a96c980eb83c4348aa9ebae18db0e1f9d55fad
+size 13882645

data_text_recog_glyph/ReCTS/data-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2142af01b48018c7c3e3312813725bf03e94b644cc7f185cff5ba219e09e0730
+size 13331324

data_text_recog_glyph/laion/data_v1.1-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9470e4fd811ab66d4daece81bab598ebab90780cb6bf38a333844859347733ec
+size 2153011640

data_text_recog_glyph/wukong_1of5/data_v1.1-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8f61d75e1d9dea8c29bb646872c52adc59b12f3c904750c25bbf930496c44f9
+size 445006180

data_text_recog_glyph/wukong_2of5/data_v1.1-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb2a5b58f11eb8ba18cdce2c52743748ccc240dadb81a8b4885b9d156a58fe2d
+size 460941784

data_text_recog_glyph/wukong_3of5/data_v1.1-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f86ed77a21e1c4ee0f08f0891064408ea9bd628508dc3033d313c35e629efa23
+size 448771446

data_text_recog_glyph/wukong_4of5/data_v1.1-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbd35b03ec9c4812bbee9e84c05f9d34d1812aca5cdb0dc3af9cf301e8455c50
+size 459732972

data_text_recog_glyph/wukong_5of5/data_v1.1-info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d2c0ed58bb69a3a01f783c25fdefc82a9f30ff2798143f4a7cc5cb021098c8f
+size 380928592

epoch_100.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7e329c97cae19e4fd3ad1b5867036952477dd09e53b90f39b2c68b100060156
+size 709732641

model_512/config.yaml

@@ -0,0 +1,46 @@
+dtype: bfloat16
+flux_path: black-forest-labs/FLUX.1-Fill-dev
+model:
+  add_cond_attn: false
+  latent_lora: false
+  union_cond_attn: true
+model_type: flux_fill
+train:
+  accumulate_grad_batches: 2
+  batch_size: 8
+  condition_type: word_fill
+  dataloader_workers: 4
+  dataset:
+    condition_size: 512
+    drop_image_prob: 0.1
+    drop_text_prob: 0.1
+    image_size: 512
+    padding: 8
+    target_size: 512
+    type: word
+  gradient_checkpointing: true
+  lora_config:
+    init_lora_weights: gaussian
+    lora_alpha: 256
+    r: 256
+    target_modules: (.*x_embedder|.*(?<!single_)transformer_blocks\.[0-9]+\.norm1\.linear|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_k|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_q|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_v|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_out\.0|.*(?<!single_)transformer_blocks\.[0-9]+\.ff\.net\.2|.*single_transformer_blocks\.[0-9]+\.norm\.linear|.*single_transformer_blocks\.[0-9]+\.proj_mlp|.*single_transformer_blocks\.[0-9]+\.proj_out|.*single_transformer_blocks\.[0-9]+\.attn.to_k|.*single_transformer_blocks\.[0-9]+\.attn.to_q|.*single_transformer_blocks\.[0-9]+\.attn.to_v|.*single_transformer_blocks\.[0-9]+\.attn.to_out)
+  max_steps: -1
+  odm_loss:
+    input_resolution: 512
+    modelpath: epoch_100.pt
+    w_loss_1: 30
+    w_loss_2: 30
+    w_loss_3: 30
+    w_loss_4: 30
+    w_loss_f: 1
+  optimizer:
+    params:
+      lr: 1
+      safeguard_warmup: true
+      use_bias_correction: true
+      weight_decay: 0.01
+    type: Prodigy
+  reuse_lora_path: reuse.safetensors
+  sample_interval: 2000
+  save_interval: 1000
+  save_path: output_dir

model_512/pytorch_lora_weights.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa64171deccf048f402a572fc6682c0c6cce7c5c462b559093cb906912e67d98
+size 1854698344

model_multisize/config.yaml

@@ -0,0 +1,59 @@
+dtype: bfloat16
+flux_path: black-forest-labs/FLUX.1-Fill-dev
+model:
+  add_cond_attn: false
+  attn_mask: false
+  latent_lora: false
+  mask_para:
+    base_ratio: 0.6
+  union_cond_attn: true
+  use_byt5_mask: false
+model_type: flux_fill
+train:
+  accumulate_grad_batches: 4
+  bucket_config:
+    1024px:
+      1:
+      - 1.0
+      - 2
+    512px:
+      1:
+      - 1.0
+      - 8
+    768px:
+      1:
+      - 1.0
+      - 4
+  condition_type: word_fill
+  dataloader_workers: 5
+  dataset:
+    drop_image_prob: 0.1
+    drop_text_prob: 0.1
+    glyph_scale: 1
+    random_select: true
+    type: word
+  gradient_checkpointing: true
+  lora_config:
+    init_lora_weights: gaussian
+    lora_alpha: 256
+    r: 256
+    target_modules: (.*x_embedder|.*(?<!single_)transformer_blocks\.[0-9]+\.norm1\.linear|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_k|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_q|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_v|.*(?<!single_)transformer_blocks\.[0-9]+\.attn\.to_out\.0|.*(?<!single_)transformer_blocks\.[0-9]+\.ff\.net\.2|.*single_transformer_blocks\.[0-9]+\.norm\.linear|.*single_transformer_blocks\.[0-9]+\.proj_mlp|.*single_transformer_blocks\.[0-9]+\.proj_out|.*single_transformer_blocks\.[0-9]+\.attn.to_k|.*single_transformer_blocks\.[0-9]+\.attn.to_q|.*single_transformer_blocks\.[0-9]+\.attn.to_v|.*single_transformer_blocks\.[0-9]+\.attn.to_out)
+  max_steps: -1
+  odm_loss:
+    modelpath: epoch_100.pt
+    w_loss_1: 20
+    w_loss_2: 20
+    w_loss_3: 20
+    w_loss_4: 20
+    w_loss_f: 1
+  optimizer:
+    params:
+      lr: 1
+      safeguard_warmup: true
+      use_bias_correction: true
+      weight_decay: 0.01
+    type: Prodigy
+  reuse_lora_path: reuse.safetensors
+  sample_interval: 2000
+  save_interval: 1000
+  save_path: output_dir

model_multisize/pytorch_lora_weights.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b92c90b4b5f0c63951f24089b454a5abb3b90cd99b49c89461f9bf92605358d
+size 1854698344