diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..0a562367e4264388f1c02f7639ea9a6377b9e6a4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,118 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +comfyui-qwen-asr/assets/intro.png filter=lfs diff=lfs merge=lfs -text +LanPaint/Example.JPG filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Flux.2.Dev_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Flux2_Klein_inpainting.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Flux_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Hunyuan_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Masked_Qwen_Image_Edit.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Masked_Qwen_Image_Edit_2509.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Qwen_Image_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Qwen_Image_Outpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/SDXL_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/wan2_2_T2I_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/wan2_2_T2I_Partial_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Z_image_base_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/example_workflows/Z_image_Inpaint.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_1/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_1/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_1/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_10/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_11/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_11/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_12/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_12/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_13/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_13/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_13/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_14/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_14/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_14/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_14/QwenEdit_2509_InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_15/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_15/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_15/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_16/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_16/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_16/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_17/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_17/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_17/Original_No_Mask.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_17/Wan22_[[:space:]]5B_Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_18/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_18/Inpainted_81frames_Drag_Me_to_ComfyUI.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_18/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_18/Original_No_Mask.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_19/Original_Load_Me_in_Loader.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_19/Outpainted_40frames_Drag_Me_to_ComfyUI.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_19/Outpainted_41frames_Drag_Me_to_ComfyUI.mp4 filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_2/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_2/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_2/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_20/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_20/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_20/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_21/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_21/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_22/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_22/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_22/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_23/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_23/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_23/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_24/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_24/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_24/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_25/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_25/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_3/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_3/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_3/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_4/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_4/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_4/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_5/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_5/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_5/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_6/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_6/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_6/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_7/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_7/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_7/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_8/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_8/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_8/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_9/InPainted_Drag_Me_to_ComfyUI.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_9/Masked_Load_Me_in_Loader.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Example_9/Original_No_Mask.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_04.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_05.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_06.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_07.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_08.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_09.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_10.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_11.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_12.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_13(1).jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_13.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_14.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_45.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/InpaintChara_46.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Inpainted_40frames_Drag_Me_to_ComfyUI_example17.gif filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Inpainted_81frames_Drag_Me_to_ComfyUI_example18.gif filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/LanPaintQwen_01.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/LanPaintQwen_03.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/LanPaintQwen_04.jpg filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Mask_Example19_.png filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Original_Load_Me_in_Loader_example19.gif filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Original_No_Mask-example18.gif filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Original_No_Mask_example17.gif filter=lfs diff=lfs merge=lfs -text +LanPaint/examples/Outpainted_40frames_Drag_Me_to_ComfyUI_example19.gif filter=lfs diff=lfs merge=lfs -text +LanPaint/Nodes.JPG filter=lfs diff=lfs merge=lfs -text diff --git a/LanPaint/.editorconfig b/LanPaint/.editorconfig new file mode 100644 index 0000000000000000000000000000000000000000..d4a2c4405ec2e962c521a13af91bf5f7098a62a8 --- /dev/null +++ b/LanPaint/.editorconfig @@ -0,0 +1,21 @@ +# http://editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 +end_of_line = lf + +[*.bat] +indent_style = tab +end_of_line = crlf + +[LICENSE] +insert_final_newline = false + +[Makefile] +indent_style = tab diff --git a/LanPaint/.github/ISSUE_TEMPLATE.md b/LanPaint/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..a35892476eb7b7e25f89cc7811ebedf3dc551d48 --- /dev/null +++ b/LanPaint/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,15 @@ +* LanPaint version: +* Python version: +* Operating System: + +### Description + +Describe what you were trying to get done. +Tell us what happened, what went wrong, and what you expected to happen. + +### What I Did + +``` +Paste the command(s) you ran and the output. +If there was a crash, please include the traceback here. +``` diff --git a/LanPaint/.github/workflows/build-pipeline.yml b/LanPaint/.github/workflows/build-pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..93b4161b6eac9efd72b13906fddbfcc0705ac366 --- /dev/null +++ b/LanPaint/.github/workflows/build-pipeline.yml @@ -0,0 +1,35 @@ +# GitHub CI build pipeline +name: LanPaint CI build + +on: + pull_request: + branches: + - master + - main +jobs: + build: + runs-on: ${{ matrix.os }} + env: + PYTHONIOENCODING: "utf8" + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev] + pip install torch --extra-index-url https://download.pytorch.org/whl/cpu + - name: Run Linting + run: | + ruff check . + - name: Run Tests + run: | + pytest tests/ diff --git a/LanPaint/.github/workflows/publish_action.yml b/LanPaint/.github/workflows/publish_action.yml new file mode 100644 index 0000000000000000000000000000000000000000..819a8d44677a402e6298ba06326cc8340c2a40ad --- /dev/null +++ b/LanPaint/.github/workflows/publish_action.yml @@ -0,0 +1,24 @@ +name: Publish to Comfy registry +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "pyproject.toml" + +permissions: + issues: write + +jobs: + publish-node: + name: Publish Custom Node to registry + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'scraed' }} + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Publish Custom Node + uses: Comfy-Org/publish-node-action@v1 + with: + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} ## Add your own personal access token to your Github Repository secrets and reference it here. \ No newline at end of file diff --git a/LanPaint/.github/workflows/publish_node.yml b/LanPaint/.github/workflows/publish_node.yml new file mode 100644 index 0000000000000000000000000000000000000000..011854235d559062b03a1c677f0c9c2636a51be8 --- /dev/null +++ b/LanPaint/.github/workflows/publish_node.yml @@ -0,0 +1,21 @@ +name: 📦 Publish to Comfy registry +on: + workflow_dispatch: + push: + tags: + - '*' + +permissions: + issues: write + +jobs: + publish-node: + name: Publish Custom Node to registry + runs-on: ubuntu-latest + steps: + - name: ♻️ Check out code + uses: actions/checkout@v4 + - name: 📦 Publish Custom Node + uses: Comfy-Org/publish-node-action@main + with: + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} diff --git a/LanPaint/.github/workflows/validate.yml b/LanPaint/.github/workflows/validate.yml new file mode 100644 index 0000000000000000000000000000000000000000..7b65b2694719e1a0d9af8fa28b106ee038003e0f --- /dev/null +++ b/LanPaint/.github/workflows/validate.yml @@ -0,0 +1,15 @@ +name: Validate backwards compatibility + +on: + pull_request: + branches: + - master + - main + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: comfy-org/node-diff@main + with: + base_ref: ${{ github.event.repository.default_branch }} diff --git a/LanPaint/.gitignore b/LanPaint/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..92436abfb848221bb57b64b4a8255d168bec4cab --- /dev/null +++ b/LanPaint/.gitignore @@ -0,0 +1,102 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# OSX useful to ignore +*.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +venv/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# IntelliJ Idea +.idea +*.iml +*.ipr +*.iws + +# PyBuilder +target/ + +# Cookiecutter +output/ +python_boilerplate/ +cookiecutter-pypackage-env/ + +# vscode settings +.history/ +*.code-workspace +.vscode/ +/.vscode diff --git a/LanPaint/.pre-commit-config.yaml b/LanPaint/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2539924861a3c53b5a1a0e8255dc92a0dd4d7223 --- /dev/null +++ b/LanPaint/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.4.9 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format diff --git a/LanPaint/.tracking b/LanPaint/.tracking new file mode 100644 index 0000000000000000000000000000000000000000..34b14b68b621dfd58e7b79643261ac16c7861e77 --- /dev/null +++ b/LanPaint/.tracking @@ -0,0 +1,155 @@ +.editorconfig +.github/ISSUE_TEMPLATE.md +.github/workflows/build-pipeline.yml +.github/workflows/publish_action.yml +.github/workflows/publish_node.yml +.github/workflows/validate.yml +.gitignore +.pre-commit-config.yaml +.vscode/settings.json +Example.JPG +LICENSE +MANIFEST.in +Nodes.JPG +README.md +__init__.py +example_workflows/Flux.2.Dev_Inpaint.jpg +example_workflows/Flux.2.Dev_Inpaint.json +example_workflows/Flux2_Klein_inpainting.jpg +example_workflows/Flux2_Klein_inpainting.json +example_workflows/Flux_Inpaint.jpg +example_workflows/Flux_Inpaint.json +example_workflows/Hunyuan_Inpaint.jpg +example_workflows/Hunyuan_Inpaint.json +example_workflows/Masked_Qwen_Image_Edit.jpg +example_workflows/Masked_Qwen_Image_Edit.json +example_workflows/Masked_Qwen_Image_Edit_2509.jpg +example_workflows/Masked_Qwen_Image_Edit_2509.json +example_workflows/Qwen_Image_Inpaint.jpg +example_workflows/Qwen_Image_Inpaint.json +example_workflows/Qwen_Image_Outpaint.jpg +example_workflows/Qwen_Image_Outpaint.json +example_workflows/SDXL_Inpaint.jpg +example_workflows/SDXL_Inpaint.json +example_workflows/Z_image_Inpaint.jpg +example_workflows/Z_image_Inpaint.json +example_workflows/Z_image_base_Inpaint.jpg +example_workflows/Z_image_base_Inpaint.json +example_workflows/wan2_2_T2I_Inpaint.jpg +example_workflows/wan2_2_T2I_Inpaint.json +example_workflows/wan2_2_T2I_Partial_Inpaint.jpg +example_workflows/wan2_2_T2I_Partial_Inpaint.json +examples/Example_1/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_1/Masked_Load_Me_in_Loader.png +examples/Example_1/Original_No_Mask.png +examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_10/Masked_Load_Me_in_Loader.png +examples/Example_11/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_11/Masked_Load_Me_in_Loader.png +examples/Example_12/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_12/Masked_Load_Me_in_Loader.png +examples/Example_13/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_13/Masked_Load_Me_in_Loader.png +examples/Example_13/Original_No_Mask.png +examples/Example_14/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_14/Masked_Load_Me_in_Loader.png +examples/Example_14/Original_No_Mask.png +examples/Example_14/QwenEdit_2509_InPainted_Drag_Me_to_ComfyUI.png +examples/Example_15/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_15/Masked_Load_Me_in_Loader.png +examples/Example_15/Original_No_Mask.png +examples/Example_16/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_16/Masked_Load_Me_in_Loader.png +examples/Example_16/Original_No_Mask.png +examples/Example_17/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 +examples/Example_17/Masked_Load_Me_in_Loader.png +examples/Example_17/Original_No_Mask.mp4 +examples/Example_17/Wan22_ 5B_Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 +examples/Example_18/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 +examples/Example_18/Inpainted_81frames_Drag_Me_to_ComfyUI.mp4 +examples/Example_18/Masked_Load_Me_in_Loader.png +examples/Example_18/Original_No_Mask.mp4 +examples/Example_19/Original_Load_Me_in_Loader.mp4 +examples/Example_19/Outpainted_40frames_Drag_Me_to_ComfyUI.mp4 +examples/Example_19/Outpainted_41frames_Drag_Me_to_ComfyUI.mp4 +examples/Example_2/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_2/Masked_Load_Me_in_Loader.png +examples/Example_2/Original_No_Mask.png +examples/Example_20/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_20/Masked_Load_Me_in_Loader.png +examples/Example_20/Original_No_Mask.png +examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_21/Masked_Load_Me_in_Loader.png +examples/Example_21/Original_No_Mask.png +examples/Example_22/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_22/Masked_Load_Me_in_Loader.png +examples/Example_22/Original_No_Mask.png +examples/Example_23/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_23/Masked_Load_Me_in_Loader.png +examples/Example_23/Original_No_Mask.png +examples/Example_24/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_24/Masked_Load_Me_in_Loader.png +examples/Example_24/Original_No_Mask.png +examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_25/Masked_Load_Me_in_Loader.png +examples/Example_25/Original_No_Mask.png +examples/Example_3/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_3/Masked_Load_Me_in_Loader.png +examples/Example_3/Original_No_Mask.png +examples/Example_4/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_4/Masked_Load_Me_in_Loader.png +examples/Example_4/Original_No_Mask.png +examples/Example_5/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_5/Masked_Load_Me_in_Loader.png +examples/Example_5/Original_No_Mask.png +examples/Example_6/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_6/Masked_Load_Me_in_Loader.png +examples/Example_6/Original_No_Mask.png +examples/Example_7/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_7/Masked_Load_Me_in_Loader.png +examples/Example_7/Original_No_Mask.png +examples/Example_8/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_8/Masked_Load_Me_in_Loader.png +examples/Example_8/Original_No_Mask.png +examples/Example_9/InPainted_Drag_Me_to_ComfyUI.png +examples/Example_9/Masked_Load_Me_in_Loader.png +examples/Example_9/Original_No_Mask.png +examples/InpaintChara_04.jpg +examples/InpaintChara_05.jpg +examples/InpaintChara_06.jpg +examples/InpaintChara_07.jpg +examples/InpaintChara_08.jpg +examples/InpaintChara_09.jpg +examples/InpaintChara_10.jpg +examples/InpaintChara_11.jpg +examples/InpaintChara_12.jpg +examples/InpaintChara_13(1).jpg +examples/InpaintChara_13.jpg +examples/InpaintChara_14.jpg +examples/InpaintChara_45.jpg +examples/InpaintChara_46.jpg +examples/Inpainted_40frames_Drag_Me_to_ComfyUI_example17.gif +examples/Inpainted_81frames_Drag_Me_to_ComfyUI_example18.gif +examples/LanPaintQwen_01.jpg +examples/LanPaintQwen_03.jpg +examples/LanPaintQwen_04.jpg +examples/Mask_Example19_.png +examples/Original_Load_Me_in_Loader_example19.gif +examples/Original_No_Mask-example18.gif +examples/Original_No_Mask_example17.gif +examples/Outpainted_40frames_Drag_Me_to_ComfyUI_example19.gif +pyproject.toml +src/LanPaint/__init__.py +src/LanPaint/earlystop.py +src/LanPaint/lanpaint.py +src/LanPaint/nodes.py +src/LanPaint/types.py +src/LanPaint/utils.py +tests/__init__.py +tests/conftest.py +tests/pytest.ini +tests/test_LanPaint.py +tests/test_lanpaint_semantic_stop.py +tests/test_reshape_mask.py +tests/test_sho_regression.py +web/js/example.js \ No newline at end of file diff --git a/LanPaint/.vscode/settings.json b/LanPaint/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..fd95ae20718b6a9f4cb348b995d40f3d5da0e3de --- /dev/null +++ b/LanPaint/.vscode/settings.json @@ -0,0 +1,12 @@ +{ + // Required - change /PATH/TO to the absolute path to ComfyUI. Windows e.g.: D:/My Folder/ComfyUI/ + // This pulls in ComfyUI Python types for the extension. + "python.analysis.extraPaths": [ + "/PATH/TO/ComfyUI/", + "/PATH/TO/ComfyUI/custom_nodes/" + ], + "cursorpyright.analysis.extraPaths": [ + "/PATH/TO/ComfyUI/", + "/PATH/TO/ComfyUI/custom_nodes/" + ], +} diff --git a/LanPaint/Example.JPG b/LanPaint/Example.JPG new file mode 100644 index 0000000000000000000000000000000000000000..46093c55d21fbd6e4b17f98955a0d93653fda6fc --- /dev/null +++ b/LanPaint/Example.JPG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4d1cea87e113c420e58a70bababfb8b0e9f7683d6fcbed9ab3704f851dce816 +size 250211 diff --git a/LanPaint/LICENSE b/LanPaint/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f288702d2fa16d3cdf0035b15a9fcbc552cd88e7 --- /dev/null +++ b/LanPaint/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/LanPaint/MANIFEST.in b/LanPaint/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..c1819f405134199c1e9c7f0b522cc91e64221a2b --- /dev/null +++ b/LanPaint/MANIFEST.in @@ -0,0 +1,9 @@ +include LICENSE +include README.md + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif + +graft src/LanPaint/web diff --git a/LanPaint/Nodes.JPG b/LanPaint/Nodes.JPG new file mode 100644 index 0000000000000000000000000000000000000000..0457d4dd499d1b334f0faa214c6b02310289d269 --- /dev/null +++ b/LanPaint/Nodes.JPG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5371453dbf997f4684a0712e19558c7c1592fc1c7cd57321aba00975689136cb +size 121562 diff --git a/LanPaint/README.md b/LanPaint/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aaf8bdc905281f6f2d82489347854965d310df23 --- /dev/null +++ b/LanPaint/README.md @@ -0,0 +1,565 @@ +
+ +# LanPaint: Universal Inpainting Sampler with "Think Mode" +[![TMLR PDF](https://img.shields.io/badge/TMLR-PDF-8A2BE2?logo=openreview&logoColor=white)](https://openreview.net/pdf?id=JPC8JyOUSW) +[![Python Benchmark](https://img.shields.io/badge/🐍-Python_Benchmark-3776AB?logo=python)](https://github.com/scraed/LanPaintBench) +[![ComfyUI Extension](https://img.shields.io/badge/ComfyUI-Extension-7B5DFF)](https://github.com/comfyanonymous/ComfyUI) +[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-yellow?logo=huggingface&logoColor=white)](https://huggingface.co/charrywhite/LanPaint) +[![Blog](https://img.shields.io/badge/📝-Blog-9cf)](https://scraed.github.io/scraedBlog/) +[![GitHub stars](https://img.shields.io/github/stars/scraed/LanPaint)](https://github.com/scraed/LanPaint/stargazers) +[![Discord](https://img.shields.io/badge/Discord-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/yN5wYDE6W4) +
+ + +Universally applicable inpainting ability for every model. LanPaint sampler lets the model "think" through multiple iterations before denoising, enabling you to invest more computation time for superior inpainting quality. + +This is the official implementation of ["LanPaint: Training-Free Diffusion Inpainting with Asymptotically Exact and Fast Conditional Sampling"](https://arxiv.org/abs/2502.03491), accepted by TMLR. + +The repository is for ComfyUI extension. + +Diffusers Support: [LanPaint-Diffusers](https://github.com/charrywhite/LanPaint-diffusers) by [@charrywhite](https://github.com/charrywhite/) + +Benchmark code for paper reproduce: [LanPaintBench](https://github.com/scraed/LanPaintBench). + +## Citation + +``` +@article{ +zheng2025lanpaint, +title={LanPaint: Training-Free Diffusion Inpainting with Asymptotically Exact and Fast Conditional Sampling}, +author={Candi Zheng and Yuan Lan and Yang Wang}, +journal={Transactions on Machine Learning Research}, +issn={2835-8856}, +year={2025}, +url={https://openreview.net/forum?id=JPC8JyOUSW}, +note={} +} +``` +**🎉 NEW 2026: Join our discord!** + +[Join our Discord](https://discord.gg/yN5wYDE6W4) to share experiences, discuss features, and explore future development. + +**🎬 NEW: LanPaint now supports inpainting and outpainting based on Z-Image!** + +`v1.5.0` fixes an important hidden bug that reduced performance and could blur images (especially with `z-image-base`) and also boosts overall LanPaint performance across other models. + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Original_No_Mask.png) | ![Masked Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png) | + +**🎬 NEW: LanPaint now supports Z-Image-Base too!** + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Original_No_Mask.png) | ![Masked Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png) | + + +**🎬 NEW: LanPaint now supports video inpainting and outpainting based on Wan 2.2!** + +
+ +| Original Video | Mask (edit T-shirt text) | Inpainted Result | +|:--------------:|:----:|:----------------:| +| ![Original](https://github.com/scraed/LanPaint/blob/master/examples/Original_No_Mask-example18.gif) | ![Mask](https://github.com/scraed/LanPaint/blob/master/examples/Example_18/Masked_Load_Me_in_Loader.png) | ![Result](https://github.com/scraed/LanPaint/blob/master/examples/Inpainted_81frames_Drag_Me_to_ComfyUI_example18.gif) | + +*Video Inpainting Example: 81 frames with temporal consistency* + +
+ +Check our latest [Wan 2.2 Video Examples](#video-examples-beta), [Wan 2.2 Image Examples](#example-wan22-inpaintlanpaint-k-sampler-5-steps-of-thinking), and +[Qwen Image Edit 2509](#example-qwen-edit-2509-inpaint) support. + + +## Table of Contents +- [Features](#features) +- [Quickstart](#quickstart) +- [How to Use Examples](#how-to-use-examples) +- [Video Examples (Beta)](#video-examples-beta) + - [Wan 2.2 Video Inpainting](#wan-22-video-inpainting) + - [Wan 2.2 5B Video Inpainting](#wan-22-5b-video-inpainting) + - [Wan 2.2 Video Outpainting](#wan-22-video-outpainting) + - [Resource Consumption](#resource-consumption) +- [Image Examples](#image-examples) + - [Flux.2.Dev](#example-flux2dev-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Flux 2 klein](#example-flux-2-klein-inpaintlanpaint-k-sampler-2-steps-of-thinking) + - [Z-image](#example-z-image-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Z-image-base](#example-z-image-base-inpaintlanpaint-k-sampler-3-steps-of-thinking) + - [Hunyuan T2I](#example-hunyuan-t2i-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Wan 2.2 T2I](#example-wan22-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Wan 2.2 T2I with reference](#example-wan22-partial-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Qwen Image Edit 2511 2509](#example-qwen-edit-2509-inpaint) + - [Qwen Image Edit 2508](#example-qwen-edit-2508-inpaint) + - [Qwen Image](#example-qwen-image-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [HiDream](#example-hidream-inpaint-lanpaint-k-sampler-5-steps-of-thinking) + - [SD 3.5](#example-sd-35-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Flux](#example-flux-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [SDXL](#example-sdxl-0-character-consistency-side-view-generation-lanpaint-k-sampler-5-steps-of-thinking) +- [Usage](#usage) + - [Basic Sampler](#basic-sampler) + - [Advanced Sampler](#lanpaint-ksampler-advanced) + - [Tuning Guide](#lanpaint-ksampler-advanced-tuning-guide) +- [Community Showcase](#community-showcase-) +- [FAQ](#faq) +- [Updates](#updates) +- [ToDo](#todo) +- [Citation](#citation) + +## Features + +- **Universal Compatibility** – Works instantly with almost any model (**Z-image, Z-image-base, Hunyuan, Wan 2.2, Qwen Image/Edit, HiDream, SD 3.5, Flux-series, SDXL, SD 1.5 or custom LoRAs**) and ControlNet. +![Inpainting Result 13](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_13.jpg) +- **No Training Needed** – Works out of the box with your existing model. +- **Easy to Use** – Same workflow as standard ComfyUI KSampler. +- **Flexible Masking** – Supports any mask shape, size, or position for inpainting/outpainting. +- **No Workarounds** – Generates 100% new content (no blending or smoothing) without relying on partial denoising. +- **Beyond Inpainting** – You can even use it as a simple way to generate consistent characters. + +**Warning**: LanPaint has degraded performance on distillation models, such as Flux.dev, due to a similar [issue with LORA training](https://medium.com/@zhiwangshi28/why-flux-lora-so-hard-to-train-and-how-to-overcome-it-a0c70bc59eaf). Please use low flux guidance (1.0-2.0) to mitigate this [issue](https://github.com/scraed/LanPaint/issues/30). + +## Quickstart + +1. **Install ComfyUI**: Follow the official [ComfyUI installation guide](https://docs.comfy.org/get_started) to set up ComfyUI on your system. Or ensure your ComfyUI version > 0.3.11. +2. **Install ComfyUI-Manager**: Add the [ComfyUI-Manager](https://github.com/ltdrdata/ComfyUI-Manager) for easy extension management. +3. **Install LanPaint Nodes**: + - **Via ComfyUI-Manager**: Search for "[LanPaint](https://registry.comfy.org/publishers/scraed/nodes/LanPaint)" in the manager and install it directly. + - **Manually**: Click "Install via Git URL" in ComfyUI-Manager and input the GitHub repository link: + ``` + https://github.com/scraed/LanPaint.git + ``` + Alternatively, clone this repository into the `ComfyUI/custom_nodes` folder. +4. **Restart ComfyUI**: Restart ComfyUI to load the LanPaint nodes. + +Once installed, you'll find the LanPaint nodes under the "sampling" category in ComfyUI. Use them just like the default KSampler for high-quality inpainting! + + +## **How to Use Examples:** +1. Navigate to the **example** folder (i.e example_1), download all pictures. +2. Drag **InPainted_Drag_Me_to_ComfyUI.png** into ComfyUI to load the workflow. +3. Download the required model (i.e clicking **Model Used in This Example**). +4. Load the model in ComfyUI. +5. Upload **Masked_Load_Me_in_Loader.png** to the **"Load image"** node in the **"Mask image for inpainting"** group (second from left), or the **Prepare Image** node. +7. Queue the task, you will get inpainted results from LanPaint. Some example also gives you inpainted results from the following methods for comparison: + - **[VAE Encode for Inpainting](https://comfyanonymous.github.io/ComfyUI_examples/inpaint/)** + - **[Set Latent Noise Mask](https://comfyui-wiki.com/en/tutorial/basic/how-to-inpaint-an-image-in-comfyui)** + +## Video Examples (Beta) + +LanPaint now supports video inpainting with Wan 2.2, enabling you to seamlessly inpaint masked regions across video frames while maintaining temporal consistency. + +**Note:** LanPaint supports video inpainting for longer sequences (e.g., 81 frames), but processing time increases significantly (please check the [Resource Consumption](#resource-consumption) section for details) and performance may become unstable. For optimal results and stability, we recommend limiting video inpainting to **40 frames or fewer**. + +### Wan 2.2 Video Inpainting + +*Example: Wan2.2 t2v 14B, 480p video (11:6), 40 frames, LanPaint K Sampler, 2 steps of thinking* + +| Original Video | Mask (Add a white hat) | Inpainted Result | +|:--------------:|:----:|:----------------:| +| ![Original Video](https://github.com/scraed/LanPaint/blob/master/examples/Original_No_Mask_example17.gif) | ![Mask](https://github.com/scraed/LanPaint/blob/master/examples/Example_17/Masked_Load_Me_in_Loader.png) | ![Inpainted Result](https://github.com/scraed/LanPaint/blob/master/examples/Inpainted_40frames_Drag_Me_to_ComfyUI_example17.gif) | + +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_17) + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + +### Wan 2.2 5B Video Inpainting + +Similar to Wan 2.2 14B with slightly different workflow. [View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_17) + +### Wan 2.2 Video Outpainting + +Extend your videos beyond their original boundaries with LanPaint's video outpainting capability based on Wan 2.2. This feature allows you to expand the canvas of your videos while maintaining coherent motion and context. + +*Example: Wan2.2 t2v 14B, 480p video (1:1 outpaint to 11:6), 40 frames, LanPaint K Sampler, 2 steps of thinking* + +| Original Video | Mask (Expand to 880x480) | Outpainted Result | +|:--------------:|:----:|:-----------------:| +| ![Original Video](https://github.com/scraed/LanPaint/blob/master/examples/Original_Load_Me_in_Loader_example19.gif) | ![Mask](https://github.com/scraed/LanPaint/blob/master/examples/Mask_Example19_.png) | ![Outpainted Result](https://github.com/scraed/LanPaint/blob/master/examples/Outpainted_40frames_Drag_Me_to_ComfyUI_example19.gif) | + +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_19) + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + +### Resource Consumption + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Processing ModeResolutionFrames ProcessedVRAM RequiredTotal Runtime (20 steps)
Inpainting880×480 (11:6)40 frames39.8 GB05:37 min
Inpainting480×480 (1:1)40 frames38.0 GB05:35 min
Outpainting880×480 (11:6)40 frames40.2 GB05:36 min
Inpainting880×480 (11:6)81 frames43.3 GB16:23 min
Inpainting480×480 (1:1)81 frames39.8 GB14:25 min
Outpainting880×480 (11:6)81 frames42.6 GB13:46 min
+ +**Test Platform**: All tests were conducted on an NVIDIA RTX Pro 6000.
+**Model Used**: `wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors` and `wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors`.
+**Processing Steps**: 20 sampling steps x 2 (LanPaint steps of thinking).
+ +**Note:** Vram is required by the model, not LanPaint. To further reduce VRAM requirements, we recommend generating less frames and loading CLIP on CPU. + +## Image Examples + +### Example Hunyuan T2I: InPaint(LanPaint K Sampler, 5 steps of thinking) +We are excited to announce that LanPaint now supports inpainting with Hunyuan text to image generation. + +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_20) + + +You need to follow the ComfyUI version of [Hunyuan workflow](https://docs.comfy.org/tutorials/video/hunyuan-video#hunyuan-text-to-video-workflow) to download and install the model. + +### Example Wan2.2: InPaint(LanPaint K Sampler, 5 steps of thinking) +We are excited to announce that LanPaint now supports Wan2.2 text to image generation with Wan2.2 T2V model. + +![Inpainting Result 45](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_45.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_15) + + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + +### Example Z-image: InPaint(LanPaint K Sampler, 5 steps of thinking) +LanPaint also supports inpainting with the Z-image text-to-image model. + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Original_No_Mask.png) | ![Masked Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_21) + +
+View Z-image Outpainting (Original / Masked / Outpainted) + +| Original | Masked | Outpainted | +|:--------:|:------:|:----------:| +| ![Original Z-image Outpaint](https://github.com/scraed/LanPaint/blob/master/examples/Example_22/Original_No_Mask.png) | ![Masked Z-image Outpaint](https://github.com/scraed/LanPaint/blob/master/examples/Example_22/Masked_Load_Me_in_Loader.png) | ![Outpainted Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_22/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Outpaint Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_22) + +You can download the Z-image model for ComfyUI from [Z-image](https://docs.comfy.org/zh-CN/tutorials/image/z-image/z-image-turbo). + +### Example Z-image-base: InPaint(LanPaint K Sampler, 3 steps of thinking) +LanPaint also supports inpainting with the Z-image-base model. + +**Warning (stability)**: Z-image-base can easily diverge with LanPaint. Start with **small `LanPaint_StepSize`** and **fewer thinking iterations** (lower `LanPaint_NumSteps`) and increase gradually only if stable. + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Original_No_Mask.png) | ![Masked Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_25) + +Workflow template (JSON): [Z_image_base_Inpaint.json](https://github.com/scraed/LanPaint/blob/master/example_workflows/Z_image_base_Inpaint.json) + +### Example Wan2.2: Partial InPaint(LanPaint K Sampler, 5 steps of thinking) +Sometimes we don't want to inpaint completely new content, but rather let the inpainted image reference the original image. One option to achieve this is to inpaint with an edit model like Qwen Image Edit. Another option is to perform a partial inpaint: allowing the diffusion process to start at some middle steps rather than from 0. + +![Inpainting Result 46](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_46.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_16) + + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + + +### Example Qwen Edit 2509: InPaint +Check our latest updated [Mased Qwen Edit Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_14) for Qwen Image Edit 2509. Download the model at [Qwen Image Edit 2509 Comfy](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models). This workflow also supports Qwen Image Edit 2511. + +![Qwen Result 3](https://github.com/scraed/LanPaint/blob/master/examples/LanPaintQwen_04.jpg) + +### Example Qwen Edit 2508: InPaint +![Qwen Result 2](https://github.com/scraed/LanPaint/blob/master/examples/LanPaintQwen_03.jpg) +Check [Mased Qwen Edit Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_14). You need to follow the ComfyUI version of [Qwen Image Edit workflow](https://docs.comfy.org/tutorials/image/qwen/qwen-image-edit) to download and install the model. + + + +### Example Qwen Image: InPaint(LanPaint K Sampler, 5 steps of thinking) + +![Inpainting Result 14](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_14.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_11) + + +You need to follow the ComfyUI version of [Qwen Image workflow](https://docs.comfy.org/tutorials/image/qwen/qwen-image) to download and install the model. + +The following examples utilize a random seed of 0 to generate a batch of 4 images for variance demonstration and fair comparison. (Note: Generating 4 images may exceed your GPU memory; please adjust the batch size as necessary.) + +![Qwen Result 1](https://github.com/scraed/LanPaint/blob/master/examples/LanPaintQwen_01.jpg) +Also check [Qwen Inpaint Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_13) and [Qwen Outpaint Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_12). You need to follow the ComfyUI version of [Qwen Image workflow](https://docs.comfy.org/tutorials/image/qwen/qwen-image) to download and install the model. + +### Example HiDream: InPaint (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_11.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_8) + +You need to follow the ComfyUI version of [HiDream workflow](https://docs.comfy.org/tutorials/image/hidream/hidream-i1) to download and install the model. + +### Example HiDream: OutPaint(LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_13(1).jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_10) + +You need to follow the ComfyUI version of [HiDream workflow](https://docs.comfy.org/tutorials/image/hidream/hidream-i1) to download and install the model. Thanks [Amazon90](https://github.com/Amazon90) for providing this example. + +### Example SD 3.5: InPaint(LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_12.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_9) + +You need to follow the ComfyUI version of [SD 3.5 workflow](https://comfyui-wiki.com/en/tutorial/advanced/stable-diffusion-3-5-comfyui-workflow) to download and install the model. + +### Example Flux.2.Dev: InPaint(LanPaint K Sampler, 5 steps of thinking) + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Flux.2.Dev](https://github.com/scraed/LanPaint/blob/master/examples/Example_23/Original_No_Mask.png) | ![Masked Flux.2.Dev](https://github.com/scraed/LanPaint/blob/master/examples/Example_23/Masked_Load_Me_in_Loader.png) | ![Inpainted Flux.2.Dev](https://github.com/scraed/LanPaint/blob/master/examples/Example_23/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_23) + +[Model Used in This Example](https://huggingface.co/Comfy-Org/flux2-dev) + +(Note: Prompt First mode is disabled on Flux.2.Dev. As it does not use CFG guidance.) + +### Example Flux 2 klein: InPaint(LanPaint K Sampler, 2 steps of thinking) + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Flux 2 klein](https://github.com/scraed/LanPaint/blob/master/examples/Example_24/Original_No_Mask.png) | ![Masked Flux 2 klein](https://github.com/scraed/LanPaint/blob/master/examples/Example_24/Masked_Load_Me_in_Loader.png) | ![Inpainted Flux 2 klein](https://github.com/scraed/LanPaint/blob/master/examples/Example_24/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_24) + +[Model Used in This Example](https://docs.comfy.org/zh-CN/tutorials/flux/flux-2-klein). If you have quality problem on Comfy 0.11 and 0.12, check [this issue](https://github.com/scraed/LanPaint/issues/80). + + +### Example Flux: InPaint(LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 7](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_10.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_7) +[Model Used in This Example](https://huggingface.co/Comfy-Org/flux1-dev/blob/main/flux1-dev-fp8.safetensors) +(Note: Prompt First mode is disabled on Flux. As it does not use CFG guidance.) + +### Example SDXL 0: Character Consistency (Side View Generation) (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 6](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_09.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_6) +[Model Used in This Example](https://civitai.com/models/1188071?modelVersionId=1408658) + +(Tricks 1: You can emphasize the character by copy it's image multiple times with Photoshop. Here I have made one extra copy.) + +(Tricks 2: Use prompts like multiple views, multiple angles, clone, turnaround. Use LanPaint's Prompt first mode (does not support Flux)) + +(Tricks 3: Remeber LanPaint can in-paint: Mask non-consistent regions and try again!) + + +### Example SDXL 1: Basket to Basket Ball (LanPaint K Sampler, 2 steps of thinking). +![Inpainting Result 1](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_04.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_1) +[Model Used in This Example](https://civitai.com/models/1188071?modelVersionId=1408658) +### Example SDXL 2: White Shirt to Blue Shirt (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 2](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_05.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_2) +[Model Used in This Example](https://civitai.com/models/1188071?modelVersionId=1408658) +### Example SDXL 3: Smile to Sad (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 3](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_06.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_3) +[Model Used in This Example](https://civitai.com/models/133005/juggernaut-xl) +### Example SDXL 4: Damage Restoration (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 4](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_07.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_4) +[Model Used in This Example](https://civitai.com/models/133005/juggernaut-xl) +### Example SDXL 5: Huge Damage Restoration (LanPaint K Sampler, 20 steps of thinking) +![Inpainting Result 5](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_08.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_5) +[Model Used in This Example](https://civitai.com/models/133005/juggernaut-xl) + +Check more for use cases like inpaint on [fine tuned models](https://github.com/scraed/LanPaint/issues/12#issuecomment-2938662021) and [face swapping](https://github.com/scraed/LanPaint/issues/12#issuecomment-2938723501), thanks to [Amazon90](https://github.com/Amazon90). + + +## Usage + +**Workflow Setup** +Same as default ComfyUI KSampler - simply replace with LanPaint KSampler nodes. The inpainting workflow is the same as the [SetLatentNoiseMask](https://comfyui-wiki.com/zh/comfyui-nodes/latent/inpaint/set-latent-noise-mask) inpainting workflow. + +**Note** +- LanPaint requires binary masks (values of 0 or 1) without opacity or smoothing. To ensure compatibility, set the mask's **opacity and hardness to maximum** in your mask editor. During inpainting, any mask with smoothing or gradients will automatically be converted to a binary mask. +- LanPaint relies heavily on your text prompts to guide inpainting - explicitly describe the content you want generated in the masked area. If results show artifacts or mismatched elements, counteract them with targeted negative prompts. + +## Basic Sampler +![Samplers](https://github.com/scraed/LanPaint/blob/master/Nodes.JPG) + +- LanPaint KSampler: The most basic and easy to use sampler for inpainting. +- LanPaint KSampler (Advanced): Full control of all parameters. + +### LanPaint KSampler +Simplified interface with recommended defaults: + +- Steps: 20 - 50. More steps will give more "thinking" and better results. +- LanPaint NumSteps: The turns of thinking before denoising. Recommend 5 for most of tasks ( which means 5 times slower than sampling without thinking). Use 10 for more challenging tasks. +- LanPaint Prompt mode: Image First mode and Prompt First mode. Image First mode focuses on the image, inpaint based on image context (maybe ignore prompt), while Prompt First mode focuses more on the prompt. Use Prompt First mode for tasks like character consistency. (Technically, it Prompt First mode change CFG scale to negative value in the BIG score to emphasis prompt, which will costs image quality.) + +### LanPaint KSampler (Advanced) +Full parameter control: +**Key Parameters** + +| Parameter | Range | Description | +|-----------|-------|-------------| +| `Steps` | 0-100 | Total steps of diffusion sampling. Higher means better inpainting. Recommend 20-50. | +| `LanPaint_NumSteps` | 0-20 | Reasoning iterations per denoising step ("thinking depth"). Easy task: 2-5. Hard task: 5-10 | +| `LanPaint_Lambda` | 0.1-50 | Content alignment strength (higher = stricter). Recommend 4.0 - 10.0 | +| `LanPaint_StepSize` | 0.1-1.0 | The StepSize of each thinking step. Recommend 0.1-0.5. | +| `LanPaint_Beta` | 0.1-2.0 | The StepSize ratio between masked / unmasked region. Small value can compensate high lambda values. Recommend 1.0 | +| `LanPaint_Friction` | 0.0-100.0 | The friction of Langevin dynamics. Higher means more slow but stable, lower means fast but unstable. Recommend 10.0 - 20.0| +| `LanPaint_EarlyStop` | 0-10 | Stop LanPaint iteration before the final sampling step. Helps to remove artifacts in some cases. Recommend 1-5| +| `LanPaint_PromptMode` | Image First / Prompt First | Image First mode focuses on the image context, maybe ignore prompt. Prompt First mode focuses more on the prompt. | + +For detailed descriptions of each parameter, simply hover your mouse over the corresponding input field to view tooltips with additional information. + +### LanPaint Mask Blend +This node blends the original image with the inpainted image based on the mask. It is useful if you want the unmasked region to match the original image pixel perfectly. + +## LanPaint KSampler (Advanced) Tuning Guide +For challenging inpainting tasks: + +1️⃣ **Boost Quality** +Increase **total number of sampling steps** (very important!), **LanPaint_NumSteps** (thinking iterations) or **LanPaint_Lambda** if the inpainted result does not meet your expectations. + +2️⃣ **Boost Speed** +Decrease **LanPaint_NumSteps** to accelerate generation! If you want better results but still need fewer steps, consider: + - **Increasing LanPaint_StepSize** to speed up the thinking process. + - **Decreasing LanPaint_Friction** to make the Langevin dynamics converges more faster. + +3️⃣ **Fix Unstability**: +If you find the results have wired texture, try +- Reduce **LanPaint_Friction** to make the Langevin dynamics more stable. +- Reduce **LanPaint_StepSize** to use smaller step size. +- Reduce **LanPaint_Beta** if you are using a high lambda value. + +⚠️ **Notes**: +- For effective tuning, **fix the seed** and adjust parameters incrementally while observing the results. This helps isolate the impact of each setting. Better to do it with a batche of images to avoid overfitting on a single image. + +## Community Showcase [](#community-showcase-) + +Discover how the community is using LanPaint! Here are some user-created tutorials: + +- [Ai绘画进阶148-三大王炸!庆祝高允贞出道6周年!T8即将直播?当AI绘画学会深度思考?!万能修复神器LanPaint,万物皆可修!-T8 Comfyui教程](https://www.youtube.com/watch?v=Z4DSTv3UPJo) +- [Ai绘画进阶151-真相了!T8竟是个AI?!LanPaint进阶(二),人物一致性,多视角实验性测试,新参数讲解,工作流分享-T8 Comfyui教程](https://www.youtube.com/watch?v=landiRhvF3k) +- [重绘和三视图角色一致性解决新方案!LanPaint节点尝试](https://www.youtube.com/watch?v=X0WbXdm6FA0) +- [ComfyUI: HiDream with Perturbation Upscale, LanPaint Inpainting (Workflow Tutorial)](https://www.youtube.com/watch?v=2-mGe4QVIIw&t=2785s) +- [ComfyUI必备LanPaint插件超详细使用教程](https://plugin.aix.ink/archives/lanpaint) + +Submit a PR to add your tutorial/video here, or open an [Issue](https://github.com/scraed/LanPaint/issues) with details! + +## FAQ +[Working togather with crop&stitch](https://github.com/scraed/LanPaint/issues/46) + +## Updates +- 2026/03/02 + - `v1.5.0`: Fixed a hidden bug that hurt performance and caused image blur (especially on `z-image-base`), and improved overall LanPaint performance on other models too. +- 2026/01/30 + - Add Z-image-base documentation and Example_25 workflow images. +- 2025/08/08 + - Add Qwen image support +- 2025/06/21 + - Update the algorithm with enhanced stability and outpaint performance. + - Add outpaint example + - Supports Sampler Custom (Thanks to [MINENEMA](https://github.com/MINENEMA)) +- 2025/06/04 + - Add more sampler support. + - Add early stopping to advanced sampler. +- 2025/05/28 + - Major update on the Langevin solver. It is now much faster and more stable. + - Greatly simplified the parameters for advanced sampler. + - Fix performance issue on Flux and SD 3.5 +- 2025/04/16 + - Added Primary HiDream support +- 2025/03/22 + - Added Primary Flux support + - Added Tease Mode +- 2025/03/10 + - LanPaint has received a major update! All examples now use the LanPaint K Sampler, offering a simplified interface with enhanced performance and stability. +- 2025/03/06: + - Bug Fix for str not callable error and unpack error. Big thanks to [jamesWalker55](https://github.com/jamesWalker55) and [EricBCoding](https://github.com/EricBCoding). + +## ToDo +- Try Implement Detailer +- ~~Provide inference code on without GUI.~~ Check our local Python benchmark code [LanPaintBench](https://github.com/scraed/LanPaintBench). + + +## Citation + +``` +@article{ +zheng2025lanpaint, +title={LanPaint: Training-Free Diffusion Inpainting with Asymptotically Exact and Fast Conditional Sampling}, +author={Candi Zheng and Yuan Lan and Yang Wang}, +journal={Transactions on Machine Learning Research}, +issn={2835-8856}, +year={2025}, +url={https://openreview.net/forum?id=JPC8JyOUSW}, +note={} +} +``` + + + + + diff --git a/LanPaint/__init__.py b/LanPaint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..87fc590e70e82c8e9b6e9688fdf497b8a2a53e5a --- /dev/null +++ b/LanPaint/__init__.py @@ -0,0 +1,94 @@ +"""Top-level package for LanPaint.""" + +__all__ = [ + "NODE_CLASS_MAPPINGS", + "NODE_DISPLAY_NAME_MAPPINGS", + "WEB_DIRECTORY", +] + +__author__ = """LanPaint""" +__email__ = "czhengac@connect.ust.hk" +__version__ = "0.0.1" + + +def _install_lightweight_runtime_stubs() -> None: + """Install lightweight stubs so tooling can import this package without ComfyUI. + + This is used by CI tooling (e.g., comfy-org/node-diff) that imports NODE_CLASS_MAPPINGS + in an environment where ComfyUI isn't installed. + """ + import sys + import types + + # `src/LanPaint/nodes.py` uses `torch.Tensor` in type annotations. + try: + import torch # noqa: F401 + except ModuleNotFoundError: + torch_mod = types.ModuleType("torch") + + class Tensor: # noqa: N801 (match torch naming) + pass + + torch_mod.Tensor = Tensor + torch_mod.nn = types.SimpleNamespace(functional=types.SimpleNamespace()) + sys.modules["torch"] = torch_mod + + if "comfyui_version" not in sys.modules: + comfyui_version_mod = types.ModuleType("comfyui_version") + comfyui_version_mod.__version__ = "0.0.0" + sys.modules["comfyui_version"] = comfyui_version_mod + + sys.modules.setdefault("nodes", types.ModuleType("nodes")) + sys.modules.setdefault("latent_preview", types.ModuleType("latent_preview")) + + if "comfy" not in sys.modules: + comfy_mod = types.ModuleType("comfy") + comfy_mod.__path__ = [] + + comfy_utils_mod = types.ModuleType("comfy.utils") + + def repeat_to_batch_size(tensor, batch_size): # type: ignore[no-untyped-def] + if getattr(tensor, "shape", ())[0] == batch_size: + return tensor + return tensor + + comfy_utils_mod.repeat_to_batch_size = repeat_to_batch_size + + comfy_samplers_mod = types.ModuleType("comfy.samplers") + + class DummyKSAMPLER: # noqa: N801 (match ComfyUI naming) + pass + + comfy_samplers_mod.KSAMPLER = DummyKSAMPLER + + comfy_model_base_mod = types.ModuleType("comfy.model_base") + + class ModelType: # noqa: N801 (match ComfyUI naming) + FLUX = "FLUX" + FLOW = "FLOW" + + class WAN22: # noqa: N801 (match ComfyUI naming) + pass + + comfy_model_base_mod.ModelType = ModelType + comfy_model_base_mod.WAN22 = WAN22 + + comfy_mod.utils = comfy_utils_mod + comfy_mod.samplers = comfy_samplers_mod + comfy_mod.model_base = comfy_model_base_mod + + sys.modules["comfy"] = comfy_mod + sys.modules["comfy.utils"] = comfy_utils_mod + sys.modules["comfy.samplers"] = comfy_samplers_mod + sys.modules["comfy.model_base"] = comfy_model_base_mod + + +try: + from .src.LanPaint.nodes import NODE_CLASS_MAPPINGS + from .src.LanPaint.nodes import NODE_DISPLAY_NAME_MAPPINGS +except ModuleNotFoundError: + _install_lightweight_runtime_stubs() + from .src.LanPaint.nodes import NODE_CLASS_MAPPINGS + from .src.LanPaint.nodes import NODE_DISPLAY_NAME_MAPPINGS + +WEB_DIRECTORY = "./web" diff --git a/LanPaint/__pycache__/__init__.cpython-312.pyc b/LanPaint/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13acf84371bf7c2d710e9c2f5c39dac5bb1089d4 Binary files /dev/null and b/LanPaint/__pycache__/__init__.cpython-312.pyc differ diff --git a/LanPaint/build/lib/LanPaint/__init__.py b/LanPaint/build/lib/LanPaint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LanPaint/build/lib/LanPaint/earlystop.py b/LanPaint/build/lib/LanPaint/earlystop.py new file mode 100644 index 0000000000000000000000000000000000000000..c1dc244d5b865d60686ce481467af6f471b2226c --- /dev/null +++ b/LanPaint/build/lib/LanPaint/earlystop.py @@ -0,0 +1,337 @@ +""" +Early Stop Logic Contributed by `https://github.com/godnight10061`. +""" + +import inspect +from typing import Any, Callable, Optional + +import torch + +from .types import LangevinState + + +def _clamp01(val: float) -> float: + if val <= 0.0: + return 0.0 + if val >= 1.0: + return 1.0 + return val + + +def _abt_scale(abt_val: float) -> float: + """ + Smooth, parameter-free scale based on outer-step noise level. + + - 0 at abt=0/1 (disable at extreme noise / extreme tail) + - 1 at abt=0.5 (mid-schedule) + """ + abt_val = _clamp01(abt_val) + return _clamp01(4.0 * abt_val * (1.0 - abt_val)) + + +def _boundary_weight(latent_mask: torch.Tensor, inpaint_weight: torch.Tensor) -> Optional[torch.Tensor]: + """ + Return a 4-neighbor boundary weight: unknown pixels adjacent to known pixels. + + This replaces the previous dilation-based "ring" (kernel/padding) and has no tunable hyperparameters. + """ + if latent_mask.dim() != 4: + return None + + known = latent_mask > 0.5 + neighbor_known = torch.zeros_like(known) + neighbor_known[:, :, 1:, :] |= known[:, :, :-1, :] + neighbor_known[:, :, :-1, :] |= known[:, :, 1:, :] + neighbor_known[:, :, :, 1:] |= known[:, :, :, :-1] + neighbor_known[:, :, :, :-1] |= known[:, :, :, 1:] + + boundary = (~known) & neighbor_known + return boundary.to(dtype=torch.float32) * inpaint_weight + + +def _weighted_mse(t1: torch.Tensor, t2: torch.Tensor, weight: torch.Tensor) -> float: + diff_sq = (t1.to(dtype=torch.float32) - t2.to(dtype=torch.float32)) ** 2 + denom = torch.sum(weight) + 1e-12 + return float((torch.sum(diff_sq * weight) / denom).item()) + + +class LanPaintEarlyStopper: + """ + Per-step early-stop logic for LanPaint inner (Langevin) iterations. + """ + + @classmethod + def from_options( + cls, + *, + model_options: Optional[dict], + latent_mask: torch.Tensor, + abt: torch.Tensor, + default_threshold: float, + default_patience: int, + default_distance_fn: Optional[Callable[..., Any]], + ) -> Optional["LanPaintEarlyStopper"]: + semantic_stop = model_options.get("lanpaint_semantic_stop") if isinstance(model_options, dict) else None + + threshold = float(default_threshold) + patience = int(default_patience) + distance_fn = default_distance_fn + # distance_fn contract: return None (use default metric) or a scalar (Python number / 0-d (1-element) torch.Tensor) + + if isinstance(semantic_stop, dict): + threshold = float(semantic_stop.get("threshold", threshold)) + patience = int(semantic_stop.get("patience", patience)) + distance_fn = semantic_stop.get("distance_fn", distance_fn) + + # Backward compatibility: map legacy 'min_steps' to a patience floor so it is not an independent knob. + if patience > 0: + min_steps = semantic_stop.get("min_steps") + if min_steps is not None: + try: + min_steps_int = int(min_steps) + except (TypeError, ValueError): + min_steps_int = 0 + if min_steps_int > 1: + patience = max(patience, min_steps_int - 1) + + enabled_early_stop = (threshold > 0.0) and (patience > 0) + # Require N+1 consecutive stable checks: + # - the first stable step sets patience_counter to 1 + # - `patience=1` therefore stops after 2 stable steps + patience_eff = max(1, patience) + 1 + threshold_eff = threshold + inpaint_weight = ring_weight = trace = abt_val = None + + if enabled_early_stop: + try: + abt_val = float(torch.mean(abt).item()) + except (TypeError, ValueError): + abt_val = 0.0 + + threshold_eff = threshold * _abt_scale(abt_val) + if threshold_eff <= 0.0: + enabled_early_stop = False + else: + inpaint_weight = (1 - latent_mask).to(dtype=torch.float32) + if float(torch.sum(inpaint_weight).item()) < 1e-6: + enabled_early_stop = False + else: + ring_weight = _boundary_weight(latent_mask, inpaint_weight) + if isinstance(model_options, dict): + trace = model_options.get("lanpaint_semantic_trace") + + if not enabled_early_stop: + return None + + # Pre-fetch trace keys to avoid repeated dict lookups + bench_case_id = bench_outer_step = bench_timestep = None + if isinstance(trace, list) and isinstance(model_options, dict): + bench_case_id = model_options.get("bench_case_id") + bench_outer_step = model_options.get("bench_outer_step") + bench_timestep = model_options.get("bench_timestep") + + return cls( + enabled=enabled_early_stop, + threshold=threshold, + threshold_eff=threshold_eff, + patience_eff=patience_eff, + inpaint_weight=inpaint_weight, + ring_weight=ring_weight, + distance_fn=distance_fn, + trace=trace, + bench_case_id=bench_case_id, + bench_outer_step=bench_outer_step, + bench_timestep=bench_timestep, + abt_val=abt_val, + ) + + def __init__( + self, + *, + enabled: bool, + threshold: float, + threshold_eff: float, + patience_eff: int, + inpaint_weight: Optional[torch.Tensor], + ring_weight: Optional[torch.Tensor], + distance_fn: Optional[Callable[..., Any]] = None, + trace: Optional[list] = None, + bench_case_id: Any = None, + bench_outer_step: Any = None, + bench_timestep: Any = None, + abt_val: Optional[float] = None, + ) -> None: + self.enabled = bool(enabled) + self.threshold = float(threshold) + self.threshold_eff = float(threshold_eff) + self.patience_eff = int(patience_eff) + + self.inpaint_weight = inpaint_weight + self.ring_weight = ring_weight + + self.trace = trace + self.bench_case_id = bench_case_id + self.bench_outer_step = bench_outer_step + self.bench_timestep = bench_timestep + self.abt_val = abt_val + + self.patience_counter = 0 + self.x0_anchor = None + + self._dist_wrapper = self._wrap_distance_fn(distance_fn) if self.enabled else None + + @property + def has_custom_distance_fn(self) -> bool: + return self._dist_wrapper is not None + + @staticmethod + def _wrap_distance_fn(distance_fn: Optional[Callable[..., Any]]): + """ + Wrap a user-provided `distance_fn` into a normalized callable: fn(prev, cur, ctx) -> dist|None. + + Supported signatures: + - 3+ positional (or *args): `distance_fn(prev, cur, ctx)` + - explicit / **kwargs ctx: `distance_fn(prev, cur, ctx=ctx)` + - default 2-arg: `distance_fn(cur, prev)` + + Return contract: None (use default metric) or a scalar (Python number / 0-d (1-element) torch.Tensor). + """ + if not callable(distance_fn): + return None + + try: + sig = inspect.signature(distance_fn) + params = list(sig.parameters.values()) + + has_ctx_param = "ctx" in sig.parameters + has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params) + has_var_pos = any(p.kind == inspect.Parameter.VAR_POSITIONAL for p in params) + + pos_params = [ + p + for p in params + if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + + if len(pos_params) >= 3 or has_var_pos: + # 3-arg positional: fn(prev, cur, ctx) + return lambda p, c, ctx: distance_fn(p, c, ctx) + if has_ctx_param or has_var_kw: + # keyword ctx: fn(prev, cur, ctx=ctx) + return lambda p, c, ctx: distance_fn(p, c, ctx=ctx) + + # Default 2-arg: fn(cur, prev) + return lambda p, c, ctx: distance_fn(c, p) + except (ValueError, TypeError): + # Fallback for built-ins or complex callables. + def fallback_wrapper(p, c, ctx): + try: + return distance_fn(p, c, ctx) + except TypeError as e: + tb = e.__traceback__ + if tb is not None and tb.tb_frame.f_code is not fallback_wrapper.__code__: + raise + return distance_fn(c, p) + + return fallback_wrapper + + def step( + self, + *, + i: int, + n_steps: int, + x_t_before: torch.Tensor, + x_t_after: torch.Tensor, + x_t_prev_for_custom: Optional[torch.Tensor], + prev_args: Any, + args: Any, + ctx: dict, + ) -> bool: + if not self.enabled: + return False + + # 'inpaint_weight' is guaranteed to be set when enabled is True in the caller. + inpaint = self.inpaint_weight + if inpaint is None: + return False + + dist = None + custom_dist = False + dist_inpaint = dist_ring = dist_drift = x0_prev = x0_cur = None + + if self._dist_wrapper is not None: + dist = self._dist_wrapper(x_t_prev_for_custom, x_t_after, ctx) + if dist is not None: + if isinstance(dist, torch.Tensor): + if dist.numel() != 1: + raise TypeError("distance_fn must return None or a scalar / 0-d (1-element) tensor") + dist = float(dist.item()) + else: + dist = float(dist) + custom_dist = dist is not None + + if dist is None: + def _get_x0(arg: Any) -> Optional[torch.Tensor]: + if isinstance(arg, LangevinState): + return arg.x0 + if isinstance(arg, tuple) and len(arg) >= 3: + return arg[2] + return None + + x0_prev = _get_x0(prev_args) + x0_cur = _get_x0(args) + + if x0_prev is not None and x0_cur is not None: + dist_inpaint = _weighted_mse(x0_cur, x0_prev, inpaint) + dist_ring = _weighted_mse(x0_cur, x0_prev, self.ring_weight) if self.ring_weight is not None else None + dist = dist_inpaint if dist_ring is None else max(dist_inpaint, dist_ring) + else: + dist_inpaint = _weighted_mse(x_t_after, x_t_before, inpaint) + dist = dist_inpaint + + threshold_used = self.threshold if custom_dist else self.threshold_eff + + # Drift guard (only for default metric with x0_cur). + if x0_cur is not None and not custom_dist: + if dist <= threshold_used: + if self.x0_anchor is None: + self.x0_anchor = x0_cur.detach() + else: + drift_inpaint = _weighted_mse(x0_cur, self.x0_anchor, inpaint) + drift_ring = _weighted_mse(x0_cur, self.x0_anchor, self.ring_weight) if self.ring_weight is not None else None + dist_drift = drift_inpaint if drift_ring is None else max(drift_inpaint, drift_ring) + dist = max(dist, dist_drift) + else: + self.x0_anchor = None + + if dist <= threshold_used: + self.patience_counter += 1 + else: + self.patience_counter = 0 + self.x0_anchor = None + + should_stop = self.patience_counter >= self.patience_eff + + if isinstance(self.trace, list): + self.trace.append( + { + "case_id": self.bench_case_id, + "outer_step": self.bench_outer_step, + "bench_timestep": self.bench_timestep, + "inner_step": i + 1, + "dist": dist, + "dist_inpaint": None if dist_inpaint is None else float(dist_inpaint), + "dist_ring": None if dist_ring is None else float(dist_ring), + "dist_drift": None if dist_drift is None else float(dist_drift), + "threshold": float(threshold_used), + "threshold_eff": float(self.threshold_eff), + "patience_counter": int(self.patience_counter), + "patience_eff": int(self.patience_eff), + "abt": None if self.abt_val is None else float(self.abt_val), + "custom_dist": bool(custom_dist), + "stopped": bool(should_stop), + } + ) + + return bool(should_stop) + diff --git a/LanPaint/build/lib/LanPaint/lanpaint.py b/LanPaint/build/lib/LanPaint/lanpaint.py new file mode 100644 index 0000000000000000000000000000000000000000..70b38d54abda0771d0f4c13a645b55f28fefe126 --- /dev/null +++ b/LanPaint/build/lib/LanPaint/lanpaint.py @@ -0,0 +1,272 @@ +import torch +from .utils import StochasticHarmonicOscillator +from functools import partial +from .earlystop import LanPaintEarlyStopper +from .types import LangevinState + +class LanPaint(): + def __init__(self, Model, NSteps, Friction, Lambda, Beta, StepSize, IS_FLUX = False, IS_FLOW = False, EarlyStopThreshold = 0.0, EarlyStopPatience = 1, EarlyStopHook = None): + self.n_steps = NSteps + self.chara_lamb = Lambda + self.IS_FLUX = IS_FLUX + self.IS_FLOW = IS_FLOW + self.step_size = StepSize + self.inner_model = Model + self.friction = Friction + self.chara_beta = Beta + self.img_dim_size = None + self.early_stop_threshold = EarlyStopThreshold + self.early_stop_patience = EarlyStopPatience + self.early_stop_hook = EarlyStopHook + + def add_none_dims(self, array): + # Create a tuple with ':' for the first dimension and 'None' repeated num_nones times + index = (slice(None),) + (None,) * (self.img_dim_size-1) + return array[index] + def remove_none_dims(self, array): + # Create a tuple with ':' for the first dimension and 'None' repeated num_nones times + index = (slice(None),) + (0,) * (self.img_dim_size-1) + return array[index] + def __call__(self, x, latent_image, noise, sigma, latent_mask, current_times, model_options, seed, n_steps=None): + self.img_dim_size = len(x.shape) + self.latent_image = latent_image + self.noise = noise + if torch.mean(torch.abs(self.noise)) < 1e-8: + self.noise = torch.randn_like(self.noise) + if n_steps is None: + n_steps = self.n_steps + return self.LanPaint(x, sigma, latent_mask, current_times, n_steps, model_options, seed, self.IS_FLUX, self.IS_FLOW) + def LanPaint(self, x, sigma, latent_mask, current_times, n_steps, model_options, seed, IS_FLUX, IS_FLOW): + input_x = x + VE_Sigma, abt, Flow_t = current_times + + step_size = self.step_size * (1 - abt) + step_size = self.add_none_dims(step_size) + # self.inner_model.inner_model.scale_latent_inpaint returns variance exploding x_t values + # This is the replace step + def scale_latent_inpaint(x, sigma, noise, latent_image): + return self.inner_model.inner_model.model_sampling.noise_scaling(sigma.reshape([sigma.shape[0]] + [1] * (len(noise.shape) - 1)), noise, latent_image) + + x = x * (1 - latent_mask) + scale_latent_inpaint(x=x, sigma=sigma, noise=self.noise, latent_image=self.latent_image)* latent_mask + + if IS_FLUX or IS_FLOW: + x_t = x * ( self.add_none_dims(abt)**0.5 + (1-self.add_none_dims(abt))**0.5 ) + else: + x_t = x / ( 1+self.add_none_dims(VE_Sigma)**2 )**0.5 # switch to variance perserving x_t values + + ############ LanPaint Iterations Start ############### + # after noise_scaling, noise = latent_image + noise * sigma, which is x_t in the variance exploding diffusion model notation for the known region. + args = None + stopper = LanPaintEarlyStopper.from_options( + model_options=model_options if isinstance(model_options, dict) else None, + latent_mask=latent_mask, + abt=abt, + default_threshold=self.early_stop_threshold, + default_patience=self.early_stop_patience, + default_distance_fn=self.early_stop_hook, + ) + + for i in range(n_steps): + score_func = partial( self.score_model, y = self.latent_image, mask = latent_mask, abt = self.add_none_dims(abt), sigma = self.add_none_dims(VE_Sigma), tflow = self.add_none_dims(Flow_t), model_options = model_options, seed = seed ) + + prev_args = args + x_t_prev = x_t.detach() if (stopper is not None and stopper.has_custom_distance_fn) else None + x_t_before = x_t if (stopper is not None and stopper.enabled) else None + + x_t, args = self.langevin_dynamics(x_t, score_func , latent_mask, step_size , current_times, sigma_x = self.add_none_dims(self.sigma_x(abt)), sigma_y = self.add_none_dims(self.sigma_y(abt)), args = args) + + if stopper is not None: + ctx = { + "step": i, + "steps_done": i + 1, + "n_steps": n_steps, + "mask": latent_mask, + "latent_image": self.latent_image, + "current_times": current_times, + "seed": seed, + } + if stopper.step( + i=i, + n_steps=n_steps, + x_t_before=x_t_before, + x_t_after=x_t, + x_t_prev_for_custom=x_t_prev, + prev_args=prev_args, + args=args, + ctx=ctx, + ): + break + + if IS_FLUX or IS_FLOW: + x = x_t / ( self.add_none_dims(abt)**0.5 + (1-self.add_none_dims(abt))**0.5 ) + else: + x = x_t * ( 1+self.add_none_dims(VE_Sigma)**2 )**0.5 # switch to variance perserving x_t values + ############ LanPaint Iterations End ############### + # out is x_0 + + out, _ = self.inner_model(x, sigma, model_options=model_options, seed=seed) + out = out * (1-latent_mask) + self.latent_image * latent_mask + + input_x.copy_(x) + return out + + def score_model(self, x_t, y, mask, abt, sigma, tflow, model_options, seed): + lamb = self.chara_lamb + if self.IS_FLUX or self.IS_FLOW: + # compute t for flow model, with a small epsilon compensating for numerical error. + x = x_t / ( abt**0.5 + (1-abt)**0.5 ) # switch to Gaussian flow matching + x_0, x_0_BIG = self.inner_model(x, self.remove_none_dims(tflow), model_options=model_options, seed=seed) + else: + x = x_t * ( 1+sigma**2 )**0.5 # switch to variance exploding + x_0, x_0_BIG = self.inner_model(x, self.remove_none_dims(sigma), model_options=model_options, seed=seed) + + score_x = -(x_t - x_0) + score_y = - (1 + lamb) * ( x_t - y ) + lamb * (x_t - x_0_BIG) + return score_x * (1 - mask) + score_y * mask + def sigma_x(self, abt): + # the time scale for the x_t update + return abt**0 + def sigma_y(self, abt): + beta = self.chara_beta * abt ** 0 + return beta + + def langevin_dynamics(self, x_t, score, mask, step_size, current_times, sigma_x=1, sigma_y=0, args=None): + if args is not None and not isinstance(args, LangevinState): + if isinstance(args, tuple): + if len(args) == 2: + # Backwards compat: older state was (v, C) without x0. + args = LangevinState(args[0], args[1], None) + elif len(args) >= 3: + args = LangevinState(args[0], args[1], args[2]) + # prepare the step size and time parameters + with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): + step_sizes = self.prepare_step_size(current_times, step_size, sigma_x, sigma_y) + sigma, abt, dtx, dty, Gamma_x, Gamma_y, A_x, A_y, D_x, D_y = step_sizes + # print('mask',mask.device) + if torch.mean(dtx) <= 0.: + return x_t, args + # ------------------------------------------------------------------------- + # Compute the Langevin dynamics update in variance perserving notation + # ------------------------------------------------------------------------- + #x0 = self.x0_evalutation(x_t, score, sigma, args) + #C = abt**0.5 * x0 / (1-abt) + A = A_x * (1-mask) + A_y * mask + D = D_x * (1-mask) + D_y * mask + dt = dtx * (1-mask) + dty * mask + Gamma = Gamma_x * (1-mask) + Gamma_y * mask + + def Coef_C(x_t): + x0 = x_t + score(x_t) + C = (abt**0.5 * x0 - x_t )/ (1-abt) + A * x_t + return C, x0 + def advance_time(x_t, v, dt, Gamma, A, C, D): + dtype = x_t.dtype + with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): + osc = StochasticHarmonicOscillator(Gamma, A, C, D ) + x_t, v = osc.dynamics(x_t, v, dt ) + x_t = x_t.to(dtype) + v = v.to(dtype) + return x_t, v + + def advance_time_overdamped(x_t, dt, A, C, D): + """ + Overdamped (Gamma -> infinity) limit: + dx = -A x dt + C dt + D dW_t + with C treated as constant over this substep. + """ + dtype = x_t.dtype + with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): + A_dt = A * dt + exp_neg = torch.exp(-A_dt) + + eps = 1e-8 + abs_A = torch.abs(A) + # k = (1 - exp(-A dt)) / A -> dt when A -> 0 + k = torch.where(abs_A < eps, dt, (-torch.expm1(-A_dt)) / A) + # k2 = (1 - exp(-2 A dt)) / (2 A) -> dt when A -> 0 + k2 = torch.where(abs_A < eps, dt, (-torch.expm1(-2 * A_dt)) / (2 * A)) + + mean = exp_neg * x_t + k * C + var = (D ** 2) * k2 + noise = torch.randn_like(x_t) * torch.sqrt(torch.clamp(var, min=0.0)) + x_t = mean + noise + return x_t.to(dtype) + + def run_damped(x_t, args): + if args is None: + v = None + C, x0 = Coef_C(x_t) + x_t, v = advance_time(x_t, v, dt, Gamma, A, C, D) + else: + v = args.v + C = args.C + x_t, v = advance_time(x_t, v, dt/2, Gamma, A, C, D) + C_new, x0 = Coef_C(x_t) + v = v + Gamma**0.5 * ( C_new - C) *dt + x_t, v = advance_time(x_t, v, dt/2, Gamma, A, C, D) + C = C_new + # args is (v, C, x0) for the next inner step. + return x_t, LangevinState(v, C, x0) + + def run_overdamped(x_t, args): + if args is None: + C, x0 = Coef_C(x_t) + x_t = advance_time_overdamped(x_t, dt, A, C, D) + else: + C = args.C + x_t = advance_time_overdamped(x_t, dt / 2, A, C, D) + C_new, x0 = Coef_C(x_t) + x_t = x_t + (C_new - C) * dt + x_t = advance_time_overdamped(x_t, dt / 2, A, C, D) + C = C_new + # args is (v, C, x0); v is None in the overdamped fallback. + return x_t, LangevinState(None, C, x0) + + try: + x_t_next, state = run_damped(x_t, args) + + v_next = state.v + if torch.isnan(x_t_next).any() or (v_next is not None and torch.isnan(v_next).any()): + raise ValueError("NaN detected") + + x_t = x_t_next + except Exception: + x_t, state = run_overdamped(x_t, args) + + # args is (v, C, x0); v can be None if we fell back to the overdamped update. + return x_t, state + + def prepare_step_size(self, current_times, step_size, sigma_x, sigma_y): + # ------------------------------------------------------------------------- + # Unpack current times parameters (sigma and abt) + sigma, abt, flow_t = current_times + sigma = self.add_none_dims(sigma) + abt = self.add_none_dims(abt) + # Compute time step (dtx, dty) for x and y branches. + dtx = 2 * step_size * sigma_x + dty = 2 * step_size * sigma_y + + # ------------------------------------------------------------------------- + # Define friction parameter Gamma_hat for each branch. + # Using dtx**0 provides a tensor of the proper device/dtype. + + Gamma_hat_x = self.friction **2 * self.step_size * sigma_x / 0.1 * sigma**0 + Gamma_hat_y = self.friction **2 * self.step_size * sigma_y / 0.1 * sigma**0 + #print("Gamma_hat_x", torch.mean(Gamma_hat_x).item(), "Gamma_hat_y", torch.mean(Gamma_hat_y).item()) + # adjust dt to match denoise-addnoise steps sizes + Gamma_hat_x /= 2. + Gamma_hat_y /= 2. + A_t_x = (1) / ( 1 - abt ) * dtx / 2 + A_t_y = (1+self.chara_lamb) / ( 1 - abt ) * dty / 2 + + + A_x = A_t_x / (dtx/2) + A_y = A_t_y / (dty/2) + Gamma_x = Gamma_hat_x / (dtx/2) + Gamma_y = Gamma_hat_y / (dty/2) + + #D_x = (2 * (1 + sigma**2) )**0.5 + #D_y = (2 * (1 + sigma**2) )**0.5 + D_x = (2 * abt**0 )**0.5 + D_y = (2 * abt**0 )**0.5 + return sigma, abt, dtx/2, dty/2, Gamma_x, Gamma_y, A_x, A_y, D_x, D_y diff --git a/LanPaint/build/lib/LanPaint/nodes.py b/LanPaint/build/lib/LanPaint/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..5c7064605b6540ebb8f2b5c5f43d4aa3a9048bd3 --- /dev/null +++ b/LanPaint/build/lib/LanPaint/nodes.py @@ -0,0 +1,648 @@ +from contextlib import contextmanager +import math +# import nodes.py +import comfy +import nodes +import latent_preview +import torch +from comfy.utils import repeat_to_batch_size +from comfy.samplers import * +from comfy.model_base import ModelType +from .lanpaint import LanPaint +from comfy.model_base import WAN22 +import comfyui_version + +def _version_tuple(value): + return tuple(int(part) if part.isdigit() else 0 for part in value.split(".")) + +COMFYUI_VERSION_060_OR_NEWER = _version_tuple(comfyui_version.__version__) >= (0, 6, 0) + +def reshape_mask(input_mask, output_shape,video_inpainting=False): + dims = len(output_shape) - 2 + print('output shape',output_shape) + scale_mode = "nearest-exact" + print('input mask',input_mask.shape,type(input_mask),torch.max(input_mask),torch.min(input_mask)) + print('target output_shape',output_shape) + print('input_mask.ndim:', input_mask.ndim, 'output_shape len:', len(output_shape)) + + # Handle input mask dimensions + if input_mask.ndim == 2: + input_mask = input_mask.unsqueeze(0).unsqueeze(0) + elif input_mask.ndim == 3: + input_mask = input_mask.unsqueeze(1) + + # Handle 5D output shape (B, C, F, H, W) by ensuring input is 5D + if len(output_shape) == 5 and input_mask.ndim == 4: + if COMFYUI_VERSION_060_OR_NEWER: + input_mask = input_mask.unsqueeze(2) # (B, C, 1, H, W) + + # Handle video case with temporal dimension + if video_inpainting: # Video case: (batch, channels, frames, height, width) + target_frames = output_shape[2] + target_height, target_width = output_shape[-2:] + + print('Video case - input_mask initial shape:', input_mask.shape) + + # First reshape input_mask to have proper dimensions for video processing + # Assume input is (frames, channels, height, width) -> (1, channels, frames, height, width) + ## if comfy version < 0.6.0 + if not COMFYUI_VERSION_060_OR_NEWER: + input_mask = input_mask.permute(1, 0, 2, 3).unsqueeze(0) + print('Video case - input_mask after reshaping:', input_mask.shape) + # Ensure we have the correct 5D shape: (batch, channels, frames, height, width) + batch_size, channels, frames, height, width = input_mask.shape + print('Video case - dimensions: batch_size={}, channels={}, frames={}, height={}, width={}'.format(batch_size, channels, frames, height, width)) + print('Video case - target size:', (target_frames, target_height, target_width)) + + # 3D nearest-exact interpolation: (batch, channels, frames, height, width) -> (batch, channels, target_frames, target_height, target_width) + temp_mask = torch.nn.functional.interpolate( + input_mask, + size=(target_frames, target_height, target_width), + mode=scale_mode, + ) + + # temp_mask is already 5D: (batch, channels, target_frames, target_height, target_width) + mask = temp_mask + print('after mask',mask.shape) + # Handle channel dimension expansion if needed + if mask.shape[1] < output_shape[1]: + mask = mask.repeat(1, output_shape[1], 1, 1, 1)[:, :output_shape[1]] + # Handle batch dimension + mask = repeat_to_batch_size(mask, output_shape[0]) + else: # Original 2D image case + if not COMFYUI_VERSION_060_OR_NEWER: + mask = torch.nn.functional.interpolate(input_mask, size=output_shape[-2:], mode=scale_mode) + else: + mask = torch.nn.functional.interpolate(input_mask, size=output_shape[2:], mode=scale_mode) + if mask.shape[1] < output_shape[1]: + mask = mask.repeat((1, output_shape[1]) + (1,) * dims)[:,:output_shape[1]] + mask = repeat_to_batch_size(mask, output_shape[0]) + + + return mask +def prepare_mask(noise_mask, shape, device,video_inpainting=False): + return reshape_mask(noise_mask, shape,video_inpainting).to(device) +def sampling_function_LanPaint(model, x, timestep, uncond, cond, cond_scale, cond_scale_BIG, model_options={}, seed=None): + if math.isclose(cond_scale, 1.0) and model_options.get("disable_cfg1_optimization", False) == False: + uncond_ = None + else: + uncond_ = uncond + + conds = [cond, uncond_] + out = calc_cond_batch(model, conds, x, timestep, model_options) + + for fn in model_options.get("sampler_pre_cfg_function", []): + args = {"conds":conds, "conds_out": out, "cond_scale": cond_scale, "timestep": timestep, + "input": x, "sigma": timestep, "model": model, "model_options": model_options} + out = fn(args) + + return cfg_function(model, out[0], out[1], cond_scale, x, timestep, model_options=model_options, cond=cond, uncond=uncond_), cfg_function(model, out[0], out[1], cond_scale_BIG, x, timestep, model_options=model_options, cond=cond, uncond=uncond_) + + +class CFGGuider_LanPaint: + def outer_sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None, **kwargs): + print("CFGGuider outer_sample") + self.inner_model, self.conds, self.loaded_models = comfy.sampler_helpers.prepare_sampling(self.model_patcher, noise.shape, self.conds, self.model_options) + device = self.model_patcher.load_device + + if isinstance(self.inner_model, WAN22): + print("WAN22 detected") + self.inner_model.extra_conds = super(WAN22, self.inner_model).extra_conds + + if denoise_mask is not None: + video_inpainting = self.model_options.get("video_inpainting", False) + denoise_mask = prepare_mask(denoise_mask, noise.shape, device, video_inpainting) + + noise = noise.to(device) + latent_image = latent_image.to(device) + sigmas = sigmas.to(device) + cast_to_load_options(self.model_options, device=device, dtype=self.model_patcher.model_dtype()) + + try: + self.model_patcher.pre_run() + output = self.inner_sample(noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, **kwargs) + finally: + self.model_patcher.cleanup() + + comfy.sampler_helpers.cleanup_models(self.conds, self.loaded_models) + del self.inner_model + del self.loaded_models + return output + def predict_noise(self, x, timestep, model_options={}, seed=None): + return sampling_function_LanPaint(self.inner_model, x, timestep, self.conds.get("negative", None), self.conds.get("positive", None), self.cfg, self.cfg_BIG, model_options=model_options, seed=seed) + +#CFGGuider.outer_sample = CFGGuider_LanPaint.outer_sample +#CFGGuider.predict_noise = CFGGuider_LanPaint.predict_noise + +class KSamplerX0Inpaint: + def __init__(self, model, sigmas): + self.inner_model = model + self.sigmas = sigmas + #self.model_sigmas = torch.cat( (torch.tensor([0.], device = sigmas.device) , torch.tensor( self.inner_model.model_patcher.get_model_object("model_sampling").sigmas, device = sigmas.device) ) ) + #self.model_sigmas = torch.tensor( self.model_sigmas, dtype = self.sigmas.dtype ) + def __call__(self, x, sigma, denoise_mask, model_options={}, seed=None,**kwargs): + ### For 1.5 and XL model + # x is x_t in the notation of variance exploding diffusion model, x_t = x_0 + sigma * noise + # sigma is the noise level + ### For flux model + # x is rectified flow x_t = sigma * noise + (1.0 - sigma) * x_0 + + IS_FLUX = self.inner_model.inner_model.model_type == ModelType.FLUX + IS_FLOW = self.inner_model.inner_model.model_type == ModelType.FLOW + #print("model class", type(self.inner_model.inner_model)) + #print("model type", self.inner_model.inner_model.model_type, "IS_FLUX", IS_FLUX, "IS_FLOW", IS_FLOW) + #print("sigma", torch.mean(sigma).item(), torch.min(sigma).item(), torch.max(sigma).item()) + # unify the notations into variance exploding diffusion model + if IS_FLUX or IS_FLOW: + Flow_t = sigma + abt = (1 - Flow_t)**2 / ((1 - Flow_t)**2 + Flow_t**2 ) + VE_Sigma = Flow_t / (1 - Flow_t) + #print("t", torch.mean( sigma ).item(), "VE_Sigma", torch.mean( VE_Sigma ).item()) + + + else: + VE_Sigma = sigma + abt = 1/( 1+VE_Sigma**2 ) + Flow_t = (1-abt)**0.5 / ( (1-abt)**0.5 + abt**0.5 ) + + if denoise_mask is not None: + if "denoise_mask_function" in model_options: + denoise_mask = model_options["denoise_mask_function"](sigma, denoise_mask, extra_options={"model": self.inner_model, "sigmas": self.sigmas}) + + denoise_mask = (denoise_mask > 0.5).float() + + latent_mask = 1 - denoise_mask + current_times = (VE_Sigma, abt, Flow_t) + + current_step = torch.argmin( torch.abs( self.sigmas - torch.mean(sigma) ) ) + total_steps = len(self.sigmas)-1 + + if total_steps - current_step <= self.LanPaint_early_stop: + out = self.PaintMethod(x, self.latent_image, self.noise, sigma, latent_mask, current_times, model_options, seed, n_steps=0) + else: + out = self.PaintMethod(x, self.latent_image, self.noise, sigma, latent_mask, current_times, model_options, seed) + else: + out, _ = self.inner_model(x, sigma, model_options=model_options, seed=seed) + + # Add TAESD preview support - directly use the latent_preview module + current_step = model_options.get("i", kwargs.get("i", 0)) + total_steps = model_options.get("total_steps", 0) + + # Only show preview every few steps to improve performance + if current_step % 2 == 0: + # Directly call the preview callback if it exists + callback = model_options.get("callback", None) + if callback is not None: + callback({"i": current_step, "denoised": out, "x": x}) + + return out + +# Custom sampler class extending ComfyUI's KSAMPLER for LanPaint +class KSAMPLER(comfy.samplers.KSAMPLER): + def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False): + #noise here is a randn noise from comfy.sample.prepare_noise + #latent_image is the latent image as input of the KSampler node. For inpainting, it is the masked latent image. Otherwise it is zero tensor. + extra_args["denoise_mask"] = denoise_mask + model_k = KSamplerX0Inpaint(model_wrap, sigmas) + model_k.latent_image = latent_image + if self.inpaint_options.get("random", False): #TODO: Should this be the default? + generator = torch.manual_seed(extra_args.get("seed", 41) + 1) + model_k.noise = torch.randn(noise.shape, generator=generator, device="cpu").to(noise.dtype).to(noise.device) + else: + model_k.noise = noise + + IS_FLUX = model_wrap.inner_model.model_type == ModelType.FLUX + IS_FLOW = model_wrap.inner_model.model_type == ModelType.FLOW + # unify the notations into variance exploding diffusion model + if IS_FLUX: + model_wrap.cfg_BIG = 1.0 + else: + model_wrap.cfg_BIG = model_wrap.model_patcher.LanPaint_cfg_BIG + noise = model_wrap.inner_model.model_sampling.noise_scaling(sigmas[0], noise, latent_image, self.max_denoise(model_wrap, sigmas)) + + model_k.PaintMethod = LanPaint(model_k.inner_model, + model_wrap.model_patcher.LanPaint_NumSteps, + model_wrap.model_patcher.LanPaint_Friction, + model_wrap.model_patcher.LanPaint_Lambda, + model_wrap.model_patcher.LanPaint_Beta, + model_wrap.model_patcher.LanPaint_StepSize, + IS_FLUX = IS_FLUX, + IS_FLOW = IS_FLOW, + EarlyStopThreshold = getattr(model_wrap.model_patcher, "LanPaint_InnerThreshold", 0.0), + EarlyStopPatience = getattr(model_wrap.model_patcher, "LanPaint_InnerPatience", 1), + EarlyStopHook = extra_args.get("model_options", {}).get("lanpaint_semantic_hook", None)) + model_k.LanPaint_early_stop = model_wrap.model_patcher.LanPaint_EarlyStop + #if not inpainting, after noise_scaling, noise = noise * sigma, which is the noise added to the clean latent image in the variance exploding diffusion model notation. + #if inpainting, after noise_scaling, noise = latent_image + noise * sigma, which is x_t in the variance exploding diffusion model notation for the known region. + k_callback = None + total_steps = len(sigmas) - 1 + if callback is not None: + k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps) + #print("LanPaint KSampler call sampler_function", self.sampler_function) + # The main loop! + #print("##########") + #print("Sampling with ", self.sampler_function) + #print("##########") + samples = self.sampler_function(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **self.extra_options) + #print("LanPaint KSampler end sampler_function") + samples = model_wrap.inner_model.model_sampling.inverse_noise_scaling(sigmas[-1], samples) + return samples + +@contextmanager +def override_sample_function(): + original_outer_sample = comfy.samplers.CFGGuider.outer_sample + comfy.samplers.CFGGuider.outer_sample = CFGGuider_LanPaint.outer_sample + + original_predict_noise = comfy.samplers.CFGGuider.predict_noise + comfy.samplers.CFGGuider.predict_noise = CFGGuider_LanPaint.predict_noise + + original_sample = comfy.samplers.KSAMPLER.sample + comfy.samplers.KSAMPLER.sample = KSAMPLER.sample + + try: + yield + finally: + comfy.samplers.KSAMPLER.sample = original_sample + comfy.samplers.CFGGuider.predict_noise = original_predict_noise + comfy.samplers.CFGGuider.outer_sample = original_outer_sample + + +class LanPaint_UpSale_LatentNoiseMask: + @classmethod + def INPUT_TYPES(s): + return {"required": { "samples": ("LATENT",), + "scale": ("INT", {"default": 2, "min": 2, "max": 8, "step": 1}), + }} + RETURN_TYPES = ("LATENT",) + FUNCTION = "set_mask" + + + CATEGORY = "latent/inpaint" + + def set_mask(self, samples, scale): + s = samples.copy() + samples = s['samples'] + # generate a mask with every scaleth pixel set to 1 + mask = torch.zeros(samples.shape[0], 1, samples.shape[2], samples.shape[3], device=samples.device) + 1 + mask[:, :, ::scale, ::scale] = 0 + s["noise_mask"] = mask + return (s,) + +#KSAMPLER_NAMES = ["euler", "dpmpp_2m", "uni_pc"] +KSAMPLER_NAMES = ["euler","euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral", + "dpm_fast", "dpmpp_sde", "dpmpp_sde_gpu", + "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", + "deis", "res_multistep", "res_multistep_ancestral", + "gradient_estimation", "er_sde", "seeds_2", "seeds_3"] + +class LanPaint_KSampler(): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL", {"tooltip": "The model used for denoising the input latent."}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "tooltip": "The random seed used for creating the noise."}), + "steps": ("INT", {"default": 30, "min": 1, "max": 10000, "tooltip": "The number of steps used in the denoising process."}), + "cfg": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01, "tooltip": "The Classifier-Free Guidance scale balances creativity and adherence to the prompt. Higher values result in images more closely matching the prompt however too high values will negatively impact quality."}), + "sampler_name": (KSAMPLER_NAMES, {"tooltip": "Recommended: euler."}), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS, {"default": "karras", "tooltip": "The scheduler controls how noise is gradually removed to form the image."}), + "positive": ("CONDITIONING", {"tooltip": "The conditioning describing the attributes you want to include in the image."}), + "negative": ("CONDITIONING", {"tooltip": "The conditioning describing the attributes you want to exclude from the image."}), + "latent_image": ("LATENT", {"tooltip": "The latent image to denoise."}), + "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "The amount of denoising applied, lower values will maintain the structure of the initial image allowing for image to image sampling."}), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "The number of steps for the Langevin dynamics, representing the turns of thinking per step."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: emphasis image quality, Prompt First: emphasis prompt following"}), + "LanPaint_Info": ("STRING", {"default": "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + "Inpainting_mode": (["🖼️ Image Inpainting", "🎬 Video Inpainting"], {"default": "🖼️ Image Inpainting", "tooltip": "Choose Image mode for photos or Video mode for video frames with temporal consistency"}), + } + } + + RETURN_TYPES = ("LATENT",) + OUTPUT_TOOLTIPS = ("The denoised latent.",) + FUNCTION = "sample" + + CATEGORY = "sampling" + DESCRIPTION = "Uses the provided model, positive and negative conditioning to denoise the latent image." + + def sample(self, model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, LanPaint_NumSteps=5, LanPaint_PromptMode="Image First", LanPaint_Info="",Inpainting_mode="🖼️ Image Inpainting"): + + model.LanPaint_StepSize = 0.2 + model.LanPaint_Lambda = 16.0 + model.LanPaint_Beta = 1. + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = 15. + model.LanPaint_EarlyStop = 1 + model.LanPaint_InnerThreshold = 0.0 + model.LanPaint_InnerPatience = 1 + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = cfg + else: + model.LanPaint_cfg_BIG = 0*cfg - 0.5 + + # Convert inpainting_mode to boolean for video_inpainting + video_inpainting = (Inpainting_mode == "🎬 Video Inpainting") + if not hasattr(model, 'model_options') or model.model_options is None: + model.model_options = {} + model.model_options["video_inpainting"] = video_inpainting + + with override_sample_function(): + return nodes.common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=denoise) +class LanPaint_KSamplerAdvanced: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model": ("MODEL",), + "add_noise": (["enable", "disable"], ), + "noise_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "steps": ("INT", {"default": 30, "min": 1, "max": 10000}), + "cfg": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}), + "sampler_name": (KSAMPLER_NAMES, ), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS, ), + "positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "latent_image": ("LATENT", ), + "start_at_step": ("INT", {"default": 0, "min": 0, "max": 10000}), + "end_at_step": ("INT", {"default": 10000, "min": 0, "max": 10000}), + "return_with_leftover_noise": (["disable", "enable"], ), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "The number of steps for the Langevin dynamics, representing the turns of thinking per step."}), + "LanPaint_Lambda": ("FLOAT", {"default": 16., "min": 0.1, "max": 50.0, "step": 0.1, "round": 0.1, "tooltip": "The bidirectional guidance scale. Higher values align with known regions more closely, but may result in instability."}), + "LanPaint_StepSize": ("FLOAT", {"default": 0.2, "min": 0.0001, "max": 1., "step": 0.01, "round": 0.001, "tooltip": "The step size for the Langevin dynamics. Higher values result in faster convergence but may be unstable."}), + "LanPaint_Beta": ("FLOAT", {"default": 1., "min": 0.0001, "max": 5, "step": 0.1, "round": 0.1, "tooltip": "The step size ratio between masked / unmasked regions. Lower value can compensate high values of LanPaint_Lambda."}), + "LanPaint_Friction": ("FLOAT", {"default": 15, "min": 0., "max": 50.0, "step": 0.1, "round": 0.1, "tooltip": "The friction parameter for fast langevin, lower values result in faster convergence but may be unstable."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: emphasis image quality, Prompt First: emphasis prompt following"}), + "LanPaint_EarlyStop": ("INT", {"default": 1, "min": 0, "max": 10000, "tooltip": "The number of steps to stop the LanPaint early, useful for preventing the image from irregular patterns."}), + "LanPaint_Info": ("STRING", {"default": "LanPaint KSampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + "Inpainting_mode": (["🖼️ Image Inpainting", "🎬 Video Inpainting"], {"default": "🖼️ Image Inpainting", "tooltip": "Choose Image mode for photos or Video mode for video frames with temporal consistency"}), + "LanPaint_InnerThreshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.0001, "round": 0.0001, "tooltip": "Early stop threshold for Langevin iterations based on semantic distance. 0.0 to disable. (Contributed by godnight10061)"}), + "LanPaint_InnerPatience": ("INT", {"default": 1, "min": 1, "max": 100, "tooltip": "Number of consecutive steps below threshold required to stop. (Contributed by godnight10061)"}), + }, + } + + RETURN_TYPES = ("LATENT",) + FUNCTION = "sample" + + CATEGORY = "sampling" + + def sample(self, model, add_noise, noise_seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, start_at_step, end_at_step, return_with_leftover_noise, LanPaint_NumSteps=5, LanPaint_Lambda=16.0, LanPaint_StepSize=0.2, LanPaint_Beta=1.0, LanPaint_Friction=15.0, LanPaint_PromptMode="Image First", LanPaint_EarlyStop=1, LanPaint_Info="", Inpainting_mode="🖼️ Image Inpainting", LanPaint_InnerThreshold=0.0, LanPaint_InnerPatience=1): + force_full_denoise = True + if return_with_leftover_noise == "enable": + force_full_denoise = False + disable_noise = False + if add_noise == "disable": + disable_noise = True + model.LanPaint_StepSize = LanPaint_StepSize + model.LanPaint_Lambda = LanPaint_Lambda + model.LanPaint_Beta = LanPaint_Beta + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = LanPaint_Friction + model.LanPaint_EarlyStop = LanPaint_EarlyStop + model.LanPaint_InnerThreshold = LanPaint_InnerThreshold + model.LanPaint_InnerPatience = LanPaint_InnerPatience + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = cfg + else: + model.LanPaint_cfg_BIG = 0*cfg - 0.5 + + # Convert inpainting_mode to boolean for video_inpainting + video_inpainting = (Inpainting_mode == "🎬 Video Inpainting") + if not hasattr(model, 'model_options') or model.model_options is None: + model.model_options = {} + model.model_options["video_inpainting"] = video_inpainting + + with override_sample_function(): + return nodes.common_ksampler(model, noise_seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, disable_noise=disable_noise, start_step=start_at_step, last_step=end_at_step, force_full_denoise=force_full_denoise) + + +class MaskBlend: + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image1": ("IMAGE", {"tooltip": "Image before inpaint"}), + "image2": ("IMAGE", {"tooltip": "Image after inpaint"}), + "mask": ("MASK",), + "blend_overlap": ("INT", {"default": 1, "min": 1, "max": 51, "step": 2, "tooltip": "The number of pixels to blend between the two images."}) + }, + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "blend_images" + + CATEGORY = "image/postprocessing" + + def blend_images(self, image1: torch.Tensor, image2: torch.Tensor, mask: torch.Tensor, blend_overlap: int): + # smooth the binary 01 mask, keep 1 still 1, but smooth the transition from 1 to 0 + # for each mask pixel, find out the nearest 1 pixel, and set the mask value to the distance between the two pixels + # check the size of mask and image1, image2, if not the same, assert error + if image1.shape[1] != image2.shape[1] or image1.shape[2] != image2.shape[2]: + raise ValueError( + "Image size mismatch: Image1 and Image2 must have the same dimensions.\n" + "Additionally, ensure both images have width and height that are multiples of 8.\n" + "This is required because VAE decode always generates images with dimensions that are multiples of 8.\n" + "If your input images are not multiples of 8, a size mismatch will occur during the decoding process.\n" + "Please resize your images using an image resize node to ensure compatibility.\n" + "Current sizes - Image1: {}x{}, Image2: {}x{}".format( + image1.shape[2], image1.shape[1], image2.shape[2], image2.shape[1] + ) + ) + mask = mask.float() + mask = torch.nn.functional.max_pool2d(mask, kernel_size=blend_overlap, stride=1, padding=blend_overlap//2) + # apply Gaussian blur with kernel size blend_overlap + kernel = self.gaussian_kernel(blend_overlap) + kernel = kernel.to(image1.device) + kernel = kernel[None, None, ...] + + mask = torch.nn.functional.conv2d(mask[:,None,:,:], kernel, padding=blend_overlap//2)[:,0,:,:] + + + blended_image = image1 * (1 - mask[...,None]) + image2 * mask[...,None] + return (blended_image,) + def gaussian_kernel(self,kernel_size): + """ + Creates a 2D Gaussian kernel with the given size and standard deviation (sigma). + """ + sigma = (kernel_size - 1)/4 + # Create a grid of (x, y) coordinates + x = torch.arange(kernel_size).float() - kernel_size // 2 + y = torch.arange(kernel_size).float() - kernel_size // 2 + x_grid, y_grid = torch.meshgrid(x, y, indexing='ij') + + # Compute the Gaussian function + kernel = torch.exp(-(x_grid ** 2 + y_grid ** 2) / (2 * sigma ** 2)) + kernel = kernel / kernel.sum() # Normalize the kernel + + return kernel + +class Noise_EmptyNoise: + def generate_noise(self, latent): + return torch.zeros_like(latent["samples"]) + +class Noise_RandomNoise: + def __init__(self, seed): + self.seed = seed + def generate_noise(self, latent): + torch.manual_seed(self.seed) + return torch.randn_like(latent["samples"]) + +# Custom sampler implementation mimmicking base comfy nodes_custom_sampler.py +class LanPaint_SamplerCustom: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model": ("MODEL",), + "add_noise": ("BOOLEAN", {"default": True}), + "noise_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "control_after_generate": True}), + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "sampler": ("SAMPLER",), + "sigmas": ("SIGMAS",), + "latent_image": ("LATENT",), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "Number of steps for Langevin dynamics, representing turns of thinking per step."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: prioritizes image quality; Prompt First: prioritizes prompt adherence."}), + "LanPaint_Info": ("STRING", {"default": "LanPaint Custom Sampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + } + } + + RETURN_TYPES = ("LATENT", "LATENT") + RETURN_NAMES = ("output", "denoised_output") + FUNCTION = "sample" + CATEGORY = "sampling/custom_sampling" + + def sample(self, model, sampler, sigmas, add_noise, noise_seed, cfg, positive, negative, latent_image, LanPaint_NumSteps, LanPaint_PromptMode, LanPaint_Info=""): + model.LanPaint_StepSize = 0.2 + model.LanPaint_Lambda = 16.0 + model.LanPaint_Beta = 1. + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = 15. + model.LanPaint_EarlyStop = 1 + model.LanPaint_InnerThreshold = 0.0 + model.LanPaint_InnerPatience = 1 + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = cfg + else: + model.LanPaint_cfg_BIG = 0 * cfg - 0.5 + with override_sample_function(): + latent = latent_image.copy() + latent_image = latent["samples"] + latent_image = comfy.sample.fix_empty_latent_channels(model, latent_image) + latent["samples"] = latent_image + + if not add_noise: + noise = Noise_EmptyNoise().generate_noise(latent) + else: + noise = Noise_RandomNoise(noise_seed).generate_noise(latent) + + noise_mask = None + if "noise_mask" in latent: + noise_mask = latent["noise_mask"] + + x0_output = {} + callback = latent_preview.prepare_callback(model, sigmas.shape[-1] - 1, x0_output) + disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED + + samples = comfy.sample.sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image,noise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=noise_seed) + + out = latent.copy() + out["samples"] = samples + if "x0" in x0_output: + out_denoised = latent.copy() + out_denoised["samples"] = model.model.process_latent_out(x0_output["x0"].cpu()) + else: + out_denoised = out + return (out, out_denoised) + +class LanPaint_SamplerCustomAdvanced: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"noise": ("NOISE",), + "guider": ("GUIDER", ), + "sampler": ("SAMPLER", ), + "sigmas": ("SIGMAS", ), + "latent_image": ("LATENT", ), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "Number of steps for Langevin dynamics, representing turns of thinking per step."}), + "LanPaint_Lambda": ("FLOAT", {"default": 16.0, "min": 0.1, "max": 50.0, "step": 0.1, "tooltip": "Bidirectional guidance scale. Higher values align with known regions but may cause instability."}), + "LanPaint_StepSize": ("FLOAT", {"default": 0.2, "min": 0.0001, "max": 1.0, "step": 0.01, "tooltip": "Step size for Langevin dynamics. Higher values speed convergence but may be unstable."}), + "LanPaint_Beta": ("FLOAT", {"default": 1.0, "min": 0.0001, "max": 5.0, "step": 0.1, "tooltip": "Step size ratio between masked/unmasked regions. Lower values balance high Lambda."}), + "LanPaint_Friction": ("FLOAT", {"default": 15.0, "min": 0.0, "max": 50.0, "step": 0.1, "tooltip": "Friction parameter for fast Langevin. Lower values speed convergence but may be unstable."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: prioritizes image quality; Prompt First: prioritizes prompt adherence."}), + "LanPaint_EarlyStop": ("INT", {"default": 1, "min": 0, "max": 10000, "tooltip": "Steps to stop LanPaint early, preventing irregular patterns."}), + "LanPaint_Info": ("STRING", {"default": "LanPaint Custom Sampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + "LanPaint_InnerThreshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.0001, "round": 0.0001, "tooltip": "Early stop threshold for Langevin iterations based on semantic distance. 0.0 to disable. (Contributed by godnight10061)"}), + "LanPaint_InnerPatience": ("INT", {"default": 1, "min": 1, "max": 100, "tooltip": "Number of consecutive steps below threshold required to stop. (Contributed by godnight10061)"}), + } + } + + RETURN_TYPES = ("LATENT","LATENT") + RETURN_NAMES = ("output", "denoised_output") + + FUNCTION = "sample" + + CATEGORY = "sampling/custom_sampling" + + def sample(self, noise, guider, sampler, sigmas, latent_image, LanPaint_NumSteps, LanPaint_Lambda, LanPaint_StepSize, LanPaint_Beta, LanPaint_Friction, LanPaint_PromptMode, LanPaint_EarlyStop, LanPaint_Info="", LanPaint_InnerThreshold=0.0, LanPaint_InnerPatience=1): + model = guider.model_patcher + model.LanPaint_StepSize = LanPaint_StepSize + model.LanPaint_Lambda = LanPaint_Lambda + model.LanPaint_Beta = LanPaint_Beta + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = LanPaint_Friction + model.LanPaint_EarlyStop = LanPaint_EarlyStop + model.LanPaint_InnerThreshold = LanPaint_InnerThreshold + model.LanPaint_InnerPatience = LanPaint_InnerPatience + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = guider.cfg + else: + model.LanPaint_cfg_BIG = 0 * guider.cfg - 0.5 + with override_sample_function(): + latent = latent_image + latent_image = latent["samples"] + latent = latent.copy() + latent_image = comfy.sample.fix_empty_latent_channels(guider.model_patcher, latent_image) + latent["samples"] = latent_image + + noise_mask = None + if "noise_mask" in latent: + noise_mask = latent["noise_mask"] + + x0_output = {} + callback = latent_preview.prepare_callback(guider.model_patcher, sigmas.shape[-1] - 1, x0_output) + + disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED + samples = guider.sample(noise.generate_noise(latent), latent_image, sampler, sigmas, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=noise.seed) + samples = samples.to(comfy.model_management.intermediate_device()) + + out = latent.copy() + out["samples"] = samples + if "x0" in x0_output: + out_denoised = latent.copy() + out_denoised["samples"] = guider.model_patcher.model.process_latent_out(x0_output["x0"].cpu()) + else: + out_denoised = out + return (out, out_denoised) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "LanPaint_KSampler": LanPaint_KSampler, + "LanPaint_KSamplerAdvanced": LanPaint_KSamplerAdvanced, + "LanPaint_SamplerCustom" : LanPaint_SamplerCustom, + "LanPaint_SamplerCustomAdvanced" : LanPaint_SamplerCustomAdvanced, + "LanPaint_MaskBlend": MaskBlend, +# "LanPaint_UpSale_LatentNoiseMask": LanPaint_UpSale_LatentNoiseMask, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "LanPaint_KSampler": "LanPaint KSampler", + "LanPaint_KSamplerAdvanced": "LanPaint KSampler (Advanced)", + "LanPaint_SamplerCustom" : "LanPaint Sampler Custom", + "LanPaint_SamplerCustomAdvanced" : "LanPaint Sampler Custom (Advanced)", + "LanPaint_MaskBlend": "LanPaint Mask Blend", +# "LanPaint_UpSale_LatentNoiseMask": "LanPaint UpSale Latent Noise Mask" +} diff --git a/LanPaint/build/lib/LanPaint/types.py b/LanPaint/build/lib/LanPaint/types.py new file mode 100644 index 0000000000000000000000000000000000000000..a738a857d08623d64fe153aed731c56440bd9cb0 --- /dev/null +++ b/LanPaint/build/lib/LanPaint/types.py @@ -0,0 +1,10 @@ +from typing import NamedTuple, Optional + +import torch + + +class LangevinState(NamedTuple): + v: Optional[torch.Tensor] + C: Optional[torch.Tensor] + x0: Optional[torch.Tensor] + diff --git a/LanPaint/build/lib/LanPaint/utils.py b/LanPaint/build/lib/LanPaint/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..774ad4c7bd4980a27e6d892f7a5ed9ae70c03e25 --- /dev/null +++ b/LanPaint/build/lib/LanPaint/utils.py @@ -0,0 +1,300 @@ +import torch +def epxm1_x(x): + # Compute the (exp(x) - 1) / x term with a small value to avoid division by zero. + result = torch.special.expm1(x) / x + # replace NaN or inf values with 0 + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + mask = torch.abs(x) < 1e-2 + result = torch.where(mask, 1 + x/2. + x**2 / 6., result) + return result +def epxm1mx_x2(x): + # Compute the (exp(x) - 1 - x) / x**2 term with a small value to avoid division by zero. + result = (torch.special.expm1(x) - x) / x**2 + # replace NaN or inf values with 0 + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + mask = torch.abs(x**2) < 1e-2 + result = torch.where(mask, 1/2. + x/6 + x**2 / 24 + x**3 / 120, result) + return result + +def expm1mxmhx2_x3(x): + # Compute the (exp(x) - 1 - x - x**2 / 2) / x**3 term with a small value to avoid division by zero. + result = (torch.special.expm1(x) - x - x**2 / 2) / x**3 + # replace NaN or inf values with 0 + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + mask = torch.abs(x**3) < 1e-2 + result = torch.where(mask, 1/6 + x/24 + x**2 / 120 + x**3 / 720 + x**4 / 5040, result) + return result + +def exp_1mcosh_GD(gamma_t, delta): + """ + Compute e^(-Γt) * (1 - cosh(Γt√Δ))/ ( (Γt)**2 Δ ) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + # Main computation + is_positive = delta > 0 + sqrt_abs_delta = torch.sqrt(torch.abs(delta)) + gamma_t_sqrt_delta = gamma_t * sqrt_abs_delta + numerator_pos = torch.exp(-gamma_t) - (torch.exp(gamma_t * (sqrt_abs_delta - 1)) + torch.exp(gamma_t * (-sqrt_abs_delta - 1))) / 2 + numerator_neg = torch.exp(-gamma_t) * ( 1 - torch.cos(gamma_t * sqrt_abs_delta ) ) + numerator = torch.where(is_positive, numerator_pos, numerator_neg) + result = numerator / (delta * gamma_t**2 ) + # Handle NaN/inf cases + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + # Handle numerical instability for small delta + mask = torch.abs(gamma_t_sqrt_delta**2) < 5e-2 + taylor = ( -0.5 - gamma_t**2 / 24 * delta - gamma_t**4 / 720 * delta**2 ) * torch.exp(-gamma_t) + result = torch.where(mask, taylor, result) + return result + +def exp_sinh_GsqrtD(gamma_t, delta): + """ + Compute e^(-Γt) * sinh(Γt√Δ) / (Γt√Δ) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + # Main computation + is_positive = delta > 0 + sqrt_abs_delta = torch.sqrt(torch.abs(delta)) + gamma_t_sqrt_delta = gamma_t * sqrt_abs_delta + numerator_pos = (torch.exp(gamma_t * (sqrt_abs_delta - 1)) - torch.exp(gamma_t * (-sqrt_abs_delta - 1))) / 2 + result_pos = numerator_pos / gamma_t_sqrt_delta + result_pos = torch.where(torch.isfinite(result_pos), result_pos, torch.zeros_like(result_pos)) + + # Taylor expansion for small gamma_t_sqrt_delta + mask = torch.abs(gamma_t_sqrt_delta) < 1e-2 + taylor = ( 1 + gamma_t**2 / 6 * delta + gamma_t**4 / 120 * delta**2 ) * torch.exp(-gamma_t) + result_pos = torch.where(mask, taylor, result_pos) + + # Handle negative delta + result_neg = torch.exp(-gamma_t) * torch.special.sinc(gamma_t_sqrt_delta/torch.pi) + result = torch.where(is_positive, result_pos, result_neg) + return result + +def exp_cosh(gamma_t, delta): + """ + Compute e^(-Γt) * cosh(Γt√Δ) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + exp_1mcosh_GD_result = exp_1mcosh_GD(gamma_t, delta) # e^(-Γt) * (1 - cosh(Γt√Δ))/ ( (Γt)**2 Δ ) + result = torch.exp(-gamma_t) - gamma_t**2 * delta * exp_1mcosh_GD_result + return result +def exp_sinh_sqrtD(gamma_t, delta): + """ + Compute e^(-Γt) * sinh(Γt√Δ) / √Δ + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + Returns: + Result of the computation with numerical stability handling + """ + exp_sinh_GsqrtD_result = exp_sinh_GsqrtD(gamma_t, delta) # e^(-Γt) * sinh(Γt√Δ) / (Γt√Δ) + result = gamma_t * exp_sinh_GsqrtD_result + return result + + + +def zeta1(gamma_t, delta): + # Compute hyperbolic terms and exponential + half_gamma_t = gamma_t / 2 + exp_cosh_term = exp_cosh(half_gamma_t, delta) + exp_sinh_term = exp_sinh_sqrtD(half_gamma_t, delta) + + + # Main computation + numerator = 1 - (exp_cosh_term + exp_sinh_term) + denominator = gamma_t * (1 - delta) / 4 + result = 1 - numerator / denominator + + # Handle numerical instability + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + + # Taylor expansion for small x (similar to your epxm1Dx approach) + mask = torch.abs(denominator) < 5e-3 + term1 = epxm1_x(-gamma_t) + term2 = epxm1mx_x2(-gamma_t) + term3 = expm1mxmhx2_x3(-gamma_t) + taylor = term1 + (1/2.+ term1-3*term2)*denominator + (-1/6. + term1/2 - 4 * term2 + 10 * term3) * denominator**2 + result = torch.where(mask, taylor, result) + + return result + +def exp_cosh_minus_terms(gamma_t, delta): + """ + Compute E^(-tΓ) * (Cosh[tΓ] - 1 - (Cosh[tΓ√Δ] - 1)/Δ) / (tΓ(1 - Δ)) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + exp_term = torch.exp(-gamma_t) + # Compute individual terms + exp_cosh_term = exp_cosh(gamma_t, gamma_t**0) - exp_term # E^(-tΓ) (Cosh[tΓ] - 1) term + exp_cosh_delta_term = - gamma_t**2 * exp_1mcosh_GD(gamma_t, delta) # E^(-tΓ) (Cosh[tΓ√Δ] - 1)/Δ term + + #exp_1mcosh_GD e^(-Γt) * (1 - cosh(Γt√Δ))/ ( (Γt)**2 Δ ) + # Main computation + numerator = exp_cosh_term - exp_cosh_delta_term + denominator = gamma_t * (1 - delta) + + result = numerator / denominator + + # Handle numerical instability + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + + # Taylor expansion for small gamma_t and delta near 1 + mask = (torch.abs(denominator) < 1e-1) + exp_1mcosh_GD_term = exp_1mcosh_GD(gamma_t, delta**0) + taylor = ( + gamma_t*exp_1mcosh_GD_term + 0.5 * gamma_t * exp_sinh_GsqrtD(gamma_t, delta**0) + - denominator / 4 * ( 0.5 * exp_cosh(gamma_t, delta**0) - 4 * exp_1mcosh_GD_term - 5 /2 * exp_sinh_GsqrtD(gamma_t, delta**0) ) + ) + result = torch.where(mask, taylor, result) + + return result + + +def zeta2(gamma_t, delta): + half_gamma_t = gamma_t / 2 + return exp_sinh_GsqrtD(half_gamma_t, delta) + +def sig11(gamma_t, delta): + return 1 - torch.exp(-gamma_t) + gamma_t**2 * exp_1mcosh_GD(gamma_t, delta) + exp_sinh_sqrtD(gamma_t, delta) + + +def Zcoefs(gamma_t, delta): + Zeta1 = zeta1(gamma_t, delta) + Zeta2 = zeta2(gamma_t, delta) + + sq_total = 1 - Zeta1 + gamma_t * (delta - 1) * (Zeta1 - 1)**2 / 8 + amplitude = torch.sqrt(sq_total) + Zcoef1 = ( gamma_t**0.5 * Zeta2 / 2 **0.5 ) / amplitude + Zcoef2 = Zcoef1 * gamma_t *( - 2 * exp_1mcosh_GD(gamma_t, delta) / sig11(gamma_t, delta) ) ** 0.5 + #cterm = exp_cosh_minus_terms(gamma_t, delta) + #sterm = exp_sinh_sqrtD(gamma_t, delta**0) + exp_sinh_sqrtD(gamma_t, delta) + #Zcoef3 = 2 * torch.sqrt( cterm / ( gamma_t * (1 - delta) * cterm + sterm ) ) + Zcoef3 = torch.sqrt( torch.maximum(1 - Zcoef1**2 - Zcoef2**2, sq_total.new_zeros(sq_total.shape)) ) + + return Zcoef1 * amplitude, Zcoef2 * amplitude, Zcoef3 * amplitude, amplitude + +def Zcoefs_asymp(gamma_t, delta): + A_t = (gamma_t * (1 - delta) )/4 + return epxm1_x(- 2 * A_t) + +class StochasticHarmonicOscillator: + """ + Simulates a stochastic harmonic oscillator governed by the equations: + dy(t) = q(t) dt + dq(t) = -Γ A y(t) dt + Γ C dt + Γ D dw(t) - Γ q(t) dt + + Also define v(t) = q(t) / √Γ, which is numerically more stable. + + Where: + y(t) - Position variable + q(t) - Velocity variable + Γ - Damping coefficient + A - Harmonic potential strength + C - Constant force term + D - Noise amplitude + dw(t) - Wiener process (Brownian motion) + """ + def __init__(self, Gamma, A, C, D): + self.Gamma = Gamma + self.A = A + self.C = C + self.D = D + self.Delta = 1 - 4 * A / Gamma + def sig11(self, gamma_t, delta): + return 1 - torch.exp(-gamma_t) + gamma_t**2 * exp_1mcosh_GD(gamma_t, delta) + exp_sinh_sqrtD(gamma_t, delta) + def sig22(self, gamma_t, delta): + return 1- zeta1(2*gamma_t, delta) + 2 * gamma_t * exp_1mcosh_GD(gamma_t, delta) + def dynamics(self, y0, v0, t): + """ + Calculates the position and velocity variables at time t. + + Parameters: + y0 (float): Initial position + v0 (float): Initial velocity v(0) = q(0) / √Γ + t (float): Time at which to evaluate the dynamics + Returns: + tuple: (y(t), v(t)) + """ + + dummyzero = y0.new_zeros(1) # convert scalar to tensor with same device and dtype as y0 + Delta = self.Delta + dummyzero + Gamma_hat = self.Gamma * t + dummyzero + A = self.A + dummyzero + C = self.C + dummyzero + D = self.D + dummyzero + Gamma = self.Gamma + dummyzero + zeta_1 = zeta1( Gamma_hat, Delta) + zeta_2 = zeta2( Gamma_hat, Delta) + EE = 1 - Gamma_hat * zeta_2 + + if v0 is None: + v0 = torch.randn_like(y0) * D / 2 ** 0.5 + #v0 = (C - A * y0)/Gamma**0.5 + + # Calculate mean position and velocity + term1 = (1 - zeta_1) * (C * t - A * t * y0) + zeta_2 * (Gamma ** 0.5) * v0 * t + y_mean = term1 + y0 + v_mean = (1 - EE)*(C - A * y0) / (Gamma ** 0.5) + (EE - A * t * (1 - zeta_1)) * v0 + + cov_yy = D**2 * t * self.sig22(Gamma_hat, Delta) + cov_vv = D**2 * self.sig11(Gamma_hat, Delta) / 2 + cov_yv = (zeta2(Gamma_hat, Delta) * Gamma_hat * D ) **2 / 2 / (Gamma ** 0.5) + + # sample new position and velocity with multivariate normal distribution + + batch_shape = y0.shape + cov_matrix = torch.zeros(*batch_shape, 2, 2, device=y0.device, dtype=y0.dtype) + cov_matrix[..., 0, 0] = cov_yy + cov_matrix[..., 0, 1] = cov_yv + cov_matrix[..., 1, 0] = cov_yv # symmetric + cov_matrix[..., 1, 1] = cov_vv + + + + # Compute the Cholesky decomposition to get scale_tril + #scale_tril = torch.linalg.cholesky(cov_matrix) + scale_tril = torch.zeros(*batch_shape, 2, 2, device=y0.device, dtype=y0.dtype) + tol = 1e-8 + cov_yy = torch.clamp( cov_yy, min = tol ) + sd_yy = torch.sqrt( cov_yy ) + inv_sd_yy = 1/(sd_yy) + + scale_tril[..., 0, 0] = sd_yy + scale_tril[..., 0, 1] = 0. + scale_tril[..., 1, 0] = cov_yv * inv_sd_yy + scale_tril[..., 1, 1] = torch.clamp( cov_vv - cov_yv**2 / cov_yy, min = tol ) ** 0.5 + # check if it matches torch.linalg. + #assert torch.allclose(torch.linalg.cholesky(cov_matrix), scale_tril, atol = 1e-4, rtol = 1e-4 ) + # Sample correlated noise from multivariate normal + mean = torch.zeros(*batch_shape, 2, device=y0.device, dtype=y0.dtype) + mean[..., 0] = y_mean + mean[..., 1] = v_mean + new_yv = torch.distributions.MultivariateNormal( + loc=mean, + scale_tril=scale_tril + ).sample() + + return new_yv[...,0], new_yv[...,1] diff --git a/LanPaint/example_workflows/Flux.2.Dev_Inpaint.jpg b/LanPaint/example_workflows/Flux.2.Dev_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..164c8ed67032e434eb8d5867607b3e29138ab26e --- /dev/null +++ b/LanPaint/example_workflows/Flux.2.Dev_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c6abff8592790ad97e096538ea2c7e370c3584feb2331619a45b3bd3e96c399 +size 1690584 diff --git a/LanPaint/example_workflows/Flux.2.Dev_Inpaint.json b/LanPaint/example_workflows/Flux.2.Dev_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..c956153a96b35bd9ec6b1eb60620e68989fc9969 --- /dev/null +++ b/LanPaint/example_workflows/Flux.2.Dev_Inpaint.json @@ -0,0 +1,1886 @@ +{ + "id": "7c048efb-a059-44e2-970a-43e1eb472d0d", + "revision": 0, + "last_node_id": 96, + "last_link_id": 216, + "nodes": [ + { + "id": 66, + "type": "MarkdownNote", + "pos": [ + -1520, + -90 + ], + "size": [ + 520, + 620 + ], + "flags": { + "collapsed": false + }, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "We are using quantized weights in this workflow, the original flux 2 repo is [here](https://huggingface.co/black-forest-labs/FLUX.2-dev/)\n\nGuide: [Subgraph](https://docs.comfy.org/interface/features/subgraph)\n\n## Report issue\n\nNote: please update ComfyUI first ([guide](https://docs.comfy.org/zh-CN/installation/update_comfyui)) and prepare the required models. Desktop/Cloud ship stable builds; nightly-supported models may not be included yet, please wait for the next stable release.\n\n- Cannot run / runtime errors: [ComfyUI/issues](https://github.com/comfyanonymous/ComfyUI/issues)\n- UI / frontend issues: [ComfyUI_frontend/issues](https://github.com/Comfy-Org/ComfyUI_frontend/issues)\n- Workflow issues: [workflow_templates/issues](https://github.com/Comfy-Org/workflow_templates/issues)\n\n\n## Model links (for local user)\n\n**text_encoders**\n\n- [mistral_3_small_flux2_bf16.safetensors](https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/text_encoders/mistral_3_small_flux2_bf16.safetensors)\n\n**loras**\n\n- [Flux_2-Turbo-LoRA_comfyui.safetensors](https://huggingface.co/ByteZSzn/Flux.2-Turbo-ComfyUI/resolve/main/Flux_2-Turbo-LoRA_comfyui.safetensors)\n\n**diffusion_models**\n\n- [flux2_dev_fp8mixed.safetensors](https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/diffusion_models/flux2_dev_fp8mixed.safetensors)\n\n**vae**\n\n- [flux2-vae.safetensors](https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors)\n\n\nModel Storage Location\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ └── mistral_3_small_flux2_bf16.safetensors\n│ ├── 📂 loras/\n│ │ └── Flux_2-Turbo-LoRA_comfyui.safetensors\n│ ├── 📂 diffusion_models/\n│ │ └── flux2_dev_fp8mixed.safetensors\n│ └── 📂 vae/\n│ └── flux2-vae.safetensors\n```\n" + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 79, + "type": "UNETLoader", + "pos": [ + -401.41628465424867, + -156.8843633555234 + ], + "size": [ + 298.1818181818182, + 82 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 185 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "flux2_dev_fp8mixed.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/diffusion_models/flux2_dev_fp8mixed.safetensors", + "directory": "diffusion_models" + }, + { + "name": "flux2_dev_fp8mixed.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/diffusion_models/flux2_dev_fp8mixed.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "flux2_dev_fp8mixed.safetensors", + "default" + ] + }, + { + "id": 81, + "type": "CLIPLoader", + "pos": [ + -401.4162846542488, + -19.521307052209544 + ], + "size": [ + 298.1818181818182, + 106 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 180 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "mistral_3_small_flux2_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/text_encoders/mistral_3_small_flux2_bf16.safetensors", + "directory": "text_encoders" + }, + { + "name": "mistral_3_small_flux2_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/text_encoders/mistral_3_small_flux2_bf16.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "mistral_3_small_flux2_fp8.safetensors", + "flux2", + "default" + ] + }, + { + "id": 82, + "type": "RandomNoise", + "pos": [ + 604.667192628257, + -167.60127519776313 + ], + "size": [ + 222.3482666015625, + 82 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 191 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "RandomNoise", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 221284764102256, + "randomize" + ] + }, + { + "id": 83, + "type": "BasicGuider", + "pos": [ + 604.667192628257, + -47.60127519776313 + ], + "size": [ + 222.3482666015625, + 46 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 185 + }, + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 184 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "slot_index": 0, + "links": [ + 192 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "BasicGuider", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 84, + "type": "KSamplerSelect", + "pos": [ + 604.667192628257, + 42.39872480223687 + ], + "size": [ + 222.3482666015625, + 58 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 193 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "KSamplerSelect", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 85, + "type": "Flux2Scheduler", + "pos": [ + 599.6491345888401, + 141.28360079347743 + ], + "size": [ + 222.3482666015625, + 106 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 187 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 189 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 194 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "Flux2Scheduler", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 20, + 1248, + 832 + ] + }, + { + "id": 78, + "type": "CLIPTextEncode", + "pos": [ + 134.66719262825703, + -138.83597106805018 + ], + "size": [ + 430, + 200 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 180 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 176 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "CLIPTextEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "the woman is wearing a light green dress" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 76, + "type": "FluxGuidance", + "pos": [ + 200.2864376852714, + 104.86432229906939 + ], + "size": [ + 317.4000244140625, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 176 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 177 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "FluxGuidance", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 4 + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 86, + "type": "EmptyFlux2LatentImage", + "pos": [ + 734.667192628257, + 392.39872480223687 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 188 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 190 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "EmptyFlux2LatentImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 1248, + 832, + 1 + ] + }, + { + "id": 88, + "type": "LanPaint_SamplerCustomAdvanced", + "pos": [ + 1001.9202375599866, + -344.1504889090143 + ], + "size": [ + 400, + 568 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 191 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 192 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 193 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 194 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 205 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 196 + ] + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "7b0d144db9aca6a4601e7c179b4c8103036e6e2b", + "Node name for S&R": "LanPaint_SamplerCustomAdvanced", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 5, + 16, + 0.15, + 1, + 15, + "Image First", + 1, + "LanPaint Custom Sampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", + "🖼️ Image Inpainting" + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 2146.7738256571624, + -100.1827620657212 + ], + "size": [ + 470, + 470 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 206 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "SaveImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Flux2" + ] + }, + { + "id": 87, + "type": "VAEDecode", + "pos": [ + 1486.9921123979855, + 399.4471845577673 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 196 + }, + { + "name": "vae", + "type": "VAE", + "link": 179 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 203 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "VAEDecode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 77, + "type": "VAELoader", + "pos": [ + -401.41628465424867, + 115.11563664447654 + ], + "size": [ + 298.1818181818182, + 60.429901123046875 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 179, + 181, + 207, + 210 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors", + "directory": "vae" + }, + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "flux2-vae.safetensors" + ] + }, + { + "id": 89, + "type": "ReferenceLatent", + "pos": [ + 182.88585037431145, + 261.7022748558416 + ], + "size": [ + 204.134765625, + 46 + ], + "flags": {}, + "order": 21, + "mode": 4, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 177 + }, + { + "name": "latent", + "shape": 7, + "type": "LATENT", + "link": 201 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 184 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "ReferenceLatent", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 93, + "type": "LanPaint_MaskBlend", + "pos": [ + 1918.4282656529942, + -298.1303483829989 + ], + "size": [ + 270, + 98 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 216 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 203 + }, + { + "name": "mask", + "type": "MASK", + "link": 202 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 206 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "7b0d144db9aca6a4601e7c179b4c8103036e6e2b", + "Node name for S&R": "LanPaint_MaskBlend", + "ue_properties": { + "widget_ue_connectable": { + "blend_overlap": true + }, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 45, + "type": "ImageScaleToTotalPixels", + "pos": [ + -942.2953950329953, + 300.7707910339225 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 128 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 208, + 214 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "ImageScaleToTotalPixels", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "lanczos", + 1, + 1 + ] + }, + { + "id": 94, + "type": "VAEEncode", + "pos": [ + -904.8642154697659, + 545.7584311148781 + ], + "size": [ + 140, + 46 + ], + "flags": { + "collapsed": true + }, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 208 + }, + { + "name": "vae", + "type": "VAE", + "link": 207 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 209 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "VAEEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 95, + "type": "VAEDecode", + "pos": [ + -922.5241424443258, + 609.877550591736 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 209 + }, + { + "name": "vae", + "type": "VAE", + "link": 210 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 211 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "VAEDecode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 90, + "type": "GetImageSize", + "pos": [ + -914.6494032834313, + 736.2708355567737 + ], + "size": [ + 140, + 124 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 211 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 187, + 188, + 197, + 212 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 189, + 190, + 198, + 213 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.7.0", + "Node name for S&R": "GetImageSize", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "width: 1024, height: 1024\n batch size: 1" + ] + }, + { + "id": 91, + "type": "VAEEncode", + "pos": [ + -256.8351358960004, + 471.2386874112704 + ], + "size": [ + 140, + 46 + ], + "flags": { + "collapsed": true + }, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 215 + }, + { + "name": "vae", + "type": "VAE", + "link": 181 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 200 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "VAEEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 96, + "type": "ImageScale", + "pos": [ + -541.9051701853617, + 509.13038411372514 + ], + "size": [ + 269.921875, + 152.8125 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 214 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 212 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 213 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 215, + 216 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "ImageScale", + "ue_properties": { + "widget_ue_connectable": { + "width": true, + "height": true + }, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "area", + 512, + 512, + "center" + ] + }, + { + "id": 92, + "type": "SetLatentNoiseMask", + "pos": [ + -47.37803371329864, + 552.1340734941332 + ], + "size": [ + 188.1373046875, + 46 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 200 + }, + { + "name": "mask", + "type": "MASK", + "link": 199 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 201, + 205 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "SetLatentNoiseMask", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 73, + "type": "MaskToImage", + "pos": [ + -514.7896658923698, + 909.9351737467276 + ], + "size": [ + 193.2779296875, + 26 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 173 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 174 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "MaskToImage", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 74, + "type": "ImageScale", + "pos": [ + -559.3815806117882, + 1005.5195473781431 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 174 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 197 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 198 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 175 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "ImageScale", + "ue_properties": { + "widget_ue_connectable": { + "width": true, + "height": true + }, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "nearest-exact", + 512, + 512, + "center" + ] + }, + { + "id": 75, + "type": "ImageToMask", + "pos": [ + -556.7866782788742, + 1200.3106983669886 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 175 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 199, + 202 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "ImageToMask", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "red" + ] + }, + { + "id": 46, + "type": "LoadImage", + "pos": [ + -960, + -90 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 128 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 173 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.71", + "Node name for S&R": "LoadImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + }, + "image": "clipspace/clipspace-painted-masked-1768018054791.png [input]" + }, + "widgets_values": [ + "inpaint_1768016968051.png", + "image" + ] + } + ], + "links": [ + [ + 128, + 46, + 0, + 45, + 0, + "IMAGE" + ], + [ + 173, + 46, + 1, + 73, + 0, + "MASK" + ], + [ + 174, + 73, + 0, + 74, + 0, + "IMAGE" + ], + [ + 175, + 74, + 0, + 75, + 0, + "IMAGE" + ], + [ + 176, + 78, + 0, + 76, + 0, + "CONDITIONING" + ], + [ + 177, + 76, + 0, + 89, + 0, + "CONDITIONING" + ], + [ + 179, + 77, + 0, + 87, + 1, + "VAE" + ], + [ + 180, + 81, + 0, + 78, + 0, + "CLIP" + ], + [ + 181, + 77, + 0, + 91, + 1, + "VAE" + ], + [ + 184, + 89, + 0, + 83, + 1, + "CONDITIONING" + ], + [ + 185, + 79, + 0, + 83, + 0, + "MODEL" + ], + [ + 187, + 90, + 0, + 85, + 0, + "INT" + ], + [ + 188, + 90, + 0, + 86, + 0, + "INT" + ], + [ + 189, + 90, + 1, + 85, + 1, + "INT" + ], + [ + 190, + 90, + 1, + 86, + 1, + "INT" + ], + [ + 191, + 82, + 0, + 88, + 0, + "NOISE" + ], + [ + 192, + 83, + 0, + 88, + 1, + "GUIDER" + ], + [ + 193, + 84, + 0, + 88, + 2, + "SAMPLER" + ], + [ + 194, + 85, + 0, + 88, + 3, + "SIGMAS" + ], + [ + 196, + 88, + 0, + 87, + 0, + "LATENT" + ], + [ + 197, + 90, + 0, + 74, + 1, + "INT" + ], + [ + 198, + 90, + 1, + 74, + 2, + "INT" + ], + [ + 199, + 75, + 0, + 92, + 1, + "MASK" + ], + [ + 200, + 91, + 0, + 92, + 0, + "LATENT" + ], + [ + 201, + 92, + 0, + 89, + 1, + "LATENT" + ], + [ + 202, + 75, + 0, + 93, + 2, + "MASK" + ], + [ + 203, + 87, + 0, + 93, + 1, + "IMAGE" + ], + [ + 205, + 92, + 0, + 88, + 4, + "LATENT" + ], + [ + 206, + 93, + 0, + 9, + 0, + "IMAGE" + ], + [ + 207, + 77, + 0, + 94, + 1, + "VAE" + ], + [ + 208, + 45, + 0, + 94, + 0, + "IMAGE" + ], + [ + 209, + 94, + 0, + 95, + 0, + "LATENT" + ], + [ + 210, + 77, + 0, + 95, + 1, + "VAE" + ], + [ + 211, + 95, + 0, + 90, + 0, + "IMAGE" + ], + [ + 212, + 90, + 0, + 96, + 1, + "INT" + ], + [ + 213, + 90, + 1, + 96, + 2, + "INT" + ], + [ + 214, + 45, + 0, + 96, + 0, + "IMAGE" + ], + [ + 215, + 96, + 0, + 91, + 0, + "IMAGE" + ], + [ + 216, + 96, + 0, + 93, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + -408.2335481707614, + -234.8843633555234, + 318.18181818181813, + 416.0299011230469 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Custom sampler", + "bounding": [ + 594.667192628257, + -237.60127519776313, + 558.5359191894531, + 501.6 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Image size", + "bounding": [ + 594.667192628257, + 282.39872480223687, + 560, + 285.6 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Prompt", + "bounding": [ + 124.667192628257, + -237.60127519776313, + 450, + 420 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "frontendVersion": "1.35.9", + "workflowRendererVersion": "LG", + "groupNodes": {}, + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "ue_links": [], + "links_added_by_ue": [], + "ds": { + "scale": 0.5530652774381023, + "offset": [ + 394.2039301836752, + 627.2932212537274 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Flux2_Klein_inpainting.jpg b/LanPaint/example_workflows/Flux2_Klein_inpainting.jpg new file mode 100644 index 0000000000000000000000000000000000000000..efcade3c87ef2186bf7a4b6415761435b50617fb --- /dev/null +++ b/LanPaint/example_workflows/Flux2_Klein_inpainting.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:214b4ab8a7e830b2555173d7effcbe647dabfcd70c5cb2d76a87b758a8fcaeea +size 1673646 diff --git a/LanPaint/example_workflows/Flux2_Klein_inpainting.json b/LanPaint/example_workflows/Flux2_Klein_inpainting.json new file mode 100644 index 0000000000000000000000000000000000000000..d9231e2020b947077684e9a92442cfc5e527f7a5 --- /dev/null +++ b/LanPaint/example_workflows/Flux2_Klein_inpainting.json @@ -0,0 +1,2665 @@ +{ + "id": "92112d97-bb64-4b44-86f2-ea5691ef8f6e", + "revision": 0, + "last_node_id": 141, + "last_link_id": 252, + "nodes": [ + { + "id": 99, + "type": "MarkdownNote", + "pos": [ + 500, + 680 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note", + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Click on the top-right corner of the node to open the [subgraph](https://docs.comfy.org/interface/features/subgraph) \n👇" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 97, + "type": "MarkdownNote", + "pos": [ + -1257.789968845821, + 803.5275371691753 + ], + "size": [ + 610, + 650 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Guide: [Subgraph](https://docs.comfy.org/interface/features/subgraph)\n\n## Model links (for local users)\n\n**text_encoders**\n\n- [qwen_3_4b.safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/text_encoders/qwen_3_4b.safetensors)\n\n**diffusion_models**\n\n- [lux-2-klein-base-4b-fp8.safetensors](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4b-fp8/resolve/main/flux-2-klein-base-4b-fp8.safetensors)\n\n**vae**\n\n- [flux2-vae.safetensors](https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors)\n\n\nModel Storage Location\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ └── qwen_3_4b.safetensors\n│ ├── 📂 diffusion_models/\n│ │ └── lux-2-klein-base-4b-fp8.safetensors\n│ └── 📂 vae/\n│ └── flux2-vae.safetensors\n```\n\n## Report issue\n\nNote: please update ComfyUI first ([guide](https://docs.comfy.org/zh-CN/installation/update_comfyui)) and prepare the required models. Desktop/Cloud ship stable builds; nightly-supported models may not be included yet, please wait for the next stable release.\n\n- Cannot run / runtime errors: [ComfyUI/issues](https://github.com/comfyanonymous/ComfyUI/issues)\n- UI / frontend issues: [ComfyUI_frontend/issues](https://github.com/Comfy-Org/ComfyUI_frontend/issues)\n- Workflow issues: [workflow_templates/issues](https://github.com/Comfy-Org/workflow_templates/issues)\n\n" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 112, + "type": "CLIPTextEncode", + "pos": [ + 361.4681372990097, + 1043.0187170159304 + ], + "size": [ + 430, + 88 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 191 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 195 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 113, + "type": "VAELoader", + "pos": [ + -68.53186060598445, + 1032.8521075621834 + ], + "size": [ + 364.42708333333337, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 190, + 205, + 234 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "flux2-vae.safetensors" + ] + }, + { + "id": 110, + "type": "CLIPLoader", + "pos": [ + -68.53186060598445, + 862.8520792606068 + ], + "size": [ + 364.42708333333337, + 106 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 191, + 192 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "qwen_3_4b.safetensors", + "url": "https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/text_encoders/qwen_3_4b.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "qwen_3_8b_fp8mixed.safetensors", + "flux2", + "default" + ] + }, + { + "id": 136, + "type": "LanPaint_MaskBlend", + "pos": [ + 2130.366524646268, + 843.6546003113864 + ], + "size": [ + 270, + 98 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 239 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 238 + }, + { + "name": "mask", + "type": "MASK", + "link": 240 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 241 + ] + } + ], + "properties": { + "Node name for S&R": "LanPaint_MaskBlend", + "cnr_id": "LanPaint", + "ver": "7b0d144db9aca6a4601e7c179b4c8103036e6e2b", + "ue_properties": { + "widget_ue_connectable": { + "blend_overlap": true + }, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 114, + "type": "ImageScaleToTotalPixels", + "pos": [ + -46.9677698074011, + 1248.0941846161222 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 204 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 198, + 199 + ] + } + ], + "properties": { + "Node name for S&R": "ImageScaleToTotalPixels", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "nearest-exact", + 1, + 1 + ] + }, + { + "id": 116, + "type": "CFGGuider", + "pos": [ + 1015.3697546367637, + 818.5786518942696 + ], + "size": [ + 270, + 98 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 189 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 196 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 197 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 207 + ] + } + ], + "properties": { + "Node name for S&R": "CFGGuider", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 117, + "type": "KSamplerSelect", + "pos": [ + 1015.3697546367637, + 958.5786930275832 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 208 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 118, + "type": "Flux2Scheduler", + "pos": [ + 1015.3697546367637, + 1058.5786098492258 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 200 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 202 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 209 + ] + } + ], + "properties": { + "Node name for S&R": "Flux2Scheduler", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 20, + 1024, + 1024 + ] + }, + { + "id": 120, + "type": "VAEDecode", + "pos": [ + 1881.3770683600626, + 748.1665033623689 + ], + "size": [ + 220, + 46 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 251 + }, + { + "name": "vae", + "type": "VAE", + "link": 190 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 238 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 123, + "type": "GetImageSize", + "pos": [ + 499.83059550468477, + 1243.3287212023115 + ], + "size": [ + 187.5, + 124 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 199 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 200, + 201 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 202, + 203 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "width: 1024, height: 1024\n batch size: 1" + ] + }, + { + "id": 115, + "type": "RandomNoise", + "pos": [ + 1015.3697546367637, + 698.5786325108902 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 206 + ] + } + ], + "properties": { + "Node name for S&R": "RandomNoise", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 0, + "fixed" + ] + }, + { + "id": 119, + "type": "EmptyFlux2LatentImage", + "pos": [ + 769.262525556713, + 1228.1103251340592 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 201 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 203 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "EmptyFlux2LatentImage", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 135, + "type": "f07d2d08-2bc5-4dd8-a9f0-f2347c6b5cca", + "pos": [ + 4.154091859301728, + 1721.4287401481058 + ], + "size": [ + 196.265234375, + 66 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "VAE", + "link": 234 + }, + { + "name": "image", + "type": "IMAGE", + "link": 235 + }, + { + "name": "mask", + "type": "MASK", + "link": 236 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 237 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 240 + ] + }, + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 239 + ] + } + ], + "properties": { + "proxyWidgets": [], + "cnr_id": "comfy-core", + "ver": "0.9.2", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 2549.2906717451565, + 1036.4687385556058 + ], + "size": [ + 416.6666666666667, + 470 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 241 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.8.2", + "Node name for S&R": "SaveImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Flux2-Klein-4b-base" + ] + }, + { + "id": 109, + "type": "UNETLoader", + "pos": [ + -68.53186060598445, + 712.8520626074891 + ], + "size": [ + 364.42708333333337, + 82 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 189 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "lux-2-klein-base-4b-fp8.safetensors", + "url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4b-fp8/resolve/main/flux-2-klein-base-4b-fp8.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "flux-2-klein-base-9b-fp8.safetensors", + "default" + ] + }, + { + "id": 122, + "type": "6007e698-2ebd-4917-84d8-299b35d7b7ab", + "pos": [ + 1160.6499310102483, + 1732.0890781447288 + ], + "size": [ + 240, + 86 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "positive", + "name": "conditioning", + "type": "CONDITIONING", + "link": 194 + }, + { + "label": "negative", + "name": "conditioning_1", + "type": "CONDITIONING", + "link": 195 + }, + { + "name": "pixels", + "type": "IMAGE", + "link": 198 + }, + { + "name": "vae", + "type": "VAE", + "link": 205 + } + ], + "outputs": [ + { + "label": "positive", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 196 + ] + }, + { + "label": "negative", + "name": "CONDITIONING_1", + "type": "CONDITIONING", + "links": [ + 197 + ] + } + ], + "properties": { + "proxyWidgets": [], + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 111, + "type": "CLIPTextEncode", + "pos": [ + 361.4681372990097, + 713.0187167443823 + ], + "size": [ + 430, + 230 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 192 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 194 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "change building's window light color to blue" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 76, + "type": "LoadImage", + "pos": [ + -540.3505464098968, + 866.1188777911093 + ], + "size": [ + 320, + 470 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 204, + 235 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 236 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + }, + "image": "clipspace/clipspace-painted-masked-1768666568593.png [input]" + }, + "widgets_values": [ + "clipspace-painted-masked-1768666568593.png", + "image" + ] + }, + { + "id": 141, + "type": "Note", + "pos": [ + 1537.948078058678, + 1731.9861154632674 + ], + "size": [ + 352.7542257974378, + 139.78283467403935 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Bypass reference conditioning if you don't need reference" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 121, + "type": "LanPaint_SamplerCustomAdvanced", + "pos": [ + 1381.2404031217425, + 648.379755917492 + ], + "size": [ + 291.4830078125, + 568 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 206 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 207 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 208 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 209 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 237 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 251 + ] + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LanPaint_SamplerCustomAdvanced", + "cnr_id": "LanPaint", + "ver": "7b0d144db9aca6a4601e7c179b4c8103036e6e2b", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 2, + 8, + 0.2, + 1, + 15, + "Image First", + 1, + "LanPaint Custom Sampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + } + ], + "links": [ + [ + 189, + 109, + 0, + 116, + 0, + "MODEL" + ], + [ + 190, + 113, + 0, + 120, + 1, + "VAE" + ], + [ + 191, + 110, + 0, + 112, + 0, + "CLIP" + ], + [ + 192, + 110, + 0, + 111, + 0, + "CLIP" + ], + [ + 194, + 111, + 0, + 122, + 0, + "CONDITIONING" + ], + [ + 195, + 112, + 0, + 122, + 1, + "CONDITIONING" + ], + [ + 196, + 122, + 0, + 116, + 1, + "CONDITIONING" + ], + [ + 197, + 122, + 1, + 116, + 2, + "CONDITIONING" + ], + [ + 198, + 114, + 0, + 122, + 2, + "IMAGE" + ], + [ + 199, + 114, + 0, + 123, + 0, + "IMAGE" + ], + [ + 200, + 123, + 0, + 118, + 0, + "INT" + ], + [ + 201, + 123, + 0, + 119, + 0, + "INT" + ], + [ + 202, + 123, + 1, + 118, + 1, + "INT" + ], + [ + 203, + 123, + 1, + 119, + 1, + "INT" + ], + [ + 204, + 76, + 0, + 114, + 0, + "IMAGE" + ], + [ + 205, + 113, + 0, + 122, + 3, + "VAE" + ], + [ + 206, + 115, + 0, + 121, + 0, + "NOISE" + ], + [ + 207, + 116, + 0, + 121, + 1, + "GUIDER" + ], + [ + 208, + 117, + 0, + 121, + 2, + "SAMPLER" + ], + [ + 209, + 118, + 0, + 121, + 3, + "SIGMAS" + ], + [ + 234, + 113, + 0, + 135, + 0, + "VAE" + ], + [ + 235, + 76, + 0, + 135, + 1, + "IMAGE" + ], + [ + 236, + 76, + 1, + 135, + 2, + "MASK" + ], + [ + 237, + 135, + 0, + 121, + 4, + "LATENT" + ], + [ + 238, + 120, + 0, + 136, + 1, + "IMAGE" + ], + [ + 239, + 135, + 2, + 136, + 0, + "IMAGE" + ], + [ + 240, + 135, + 1, + 136, + 2, + "MASK" + ], + [ + 241, + 136, + 0, + 9, + 0, + "IMAGE" + ], + [ + 251, + 121, + 0, + 120, + 0, + "LATENT" + ] + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + -72.19852871584465, + 628.9909451488317, + 380, + 550 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompt", + "bounding": [ + 347.8014712841558, + 628.9909451488317, + 470, + 550 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Sampler", + "bounding": [ + 1015.7031229873269, + 634.5508624623236, + 532.3638671875, + 550 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "definitions": { + "subgraphs": [ + { + "id": "6007e698-2ebd-4917-84d8-299b35d7b7ab", + "version": 1, + "state": { + "lastGroupId": 4, + "lastNodeId": 78, + "lastLinkId": 170, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Reference Conditioning", + "inputNode": { + "id": -10, + "bounding": [ + -270, + 990, + 120, + 120 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 580, + 970, + 120, + 80 + ] + }, + "inputs": [ + { + "id": "5c9a0f5e-8cee-4947-90bc-330de782043a", + "name": "conditioning", + "type": "CONDITIONING", + "linkIds": [ + 165 + ], + "label": "positive", + "pos": [ + -170, + 1010 + ] + }, + { + "id": "61826d46-4c21-4ad6-801c-3e3fa94115e2", + "name": "conditioning_1", + "type": "CONDITIONING", + "linkIds": [ + 166 + ], + "label": "negative", + "pos": [ + -170, + 1030 + ] + }, + { + "id": "345bf085-5939-47ff-9767-8f8f239a719c", + "name": "pixels", + "type": "IMAGE", + "linkIds": [ + 167 + ], + "pos": [ + -170, + 1050 + ] + }, + { + "id": "f4594e34-e2f5-4f1e-b1fa-a1dc2aeb0a90", + "name": "vae", + "type": "VAE", + "linkIds": [ + 168 + ], + "pos": [ + -170, + 1070 + ] + } + ], + "outputs": [ + { + "id": "b3357c0e-6428-4055-9cd3-3595f0896fa8", + "name": "CONDITIONING", + "type": "CONDITIONING", + "linkIds": [ + 169 + ], + "label": "positive", + "pos": [ + 600, + 990 + ] + }, + { + "id": "01519713-2ed1-4694-a387-79f44e088e89", + "name": "CONDITIONING_1", + "type": "CONDITIONING", + "linkIds": [ + 170 + ], + "label": "negative", + "pos": [ + 600, + 1010 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 76, + "type": "ReferenceLatent", + "pos": [ + 170, + 1050 + ], + "size": [ + 204.134765625, + 46 + ], + "flags": { + "collapsed": false + }, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "conditioning", + "name": "conditioning", + "type": "CONDITIONING", + "link": 166 + }, + { + "localized_name": "latent", + "name": "latent", + "shape": 7, + "type": "LATENT", + "link": 163 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 170 + ] + } + ], + "properties": { + "Node name for S&R": "ReferenceLatent", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 78, + "type": "VAEEncode", + "pos": [ + -90, + 1150 + ], + "size": [ + 190, + 46 + ], + "flags": { + "collapsed": false + }, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "pixels", + "name": "pixels", + "type": "IMAGE", + "link": 167 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 168 + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 163, + 164 + ] + } + ], + "properties": { + "Node name for S&R": "VAEEncode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 77, + "type": "ReferenceLatent", + "pos": [ + 170, + 940 + ], + "size": [ + 210, + 46 + ], + "flags": { + "collapsed": false + }, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "conditioning", + "name": "conditioning", + "type": "CONDITIONING", + "link": 165 + }, + { + "localized_name": "latent", + "name": "latent", + "shape": 7, + "type": "LATENT", + "link": 164 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 169 + ] + } + ], + "properties": { + "Node name for S&R": "ReferenceLatent", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + } + ], + "groups": [], + "links": [ + { + "id": 163, + "origin_id": 78, + "origin_slot": 0, + "target_id": 76, + "target_slot": 1, + "type": "LATENT" + }, + { + "id": 164, + "origin_id": 78, + "origin_slot": 0, + "target_id": 77, + "target_slot": 1, + "type": "LATENT" + }, + { + "id": 165, + "origin_id": -10, + "origin_slot": 0, + "target_id": 77, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 166, + "origin_id": -10, + "origin_slot": 1, + "target_id": 76, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 167, + "origin_id": -10, + "origin_slot": 2, + "target_id": 78, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 168, + "origin_id": -10, + "origin_slot": 3, + "target_id": 78, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 169, + "origin_id": 77, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 170, + "origin_id": 76, + "origin_slot": 0, + "target_id": -20, + "target_slot": 1, + "type": "CONDITIONING" + } + ], + "extra": { + "workflowRendererVersion": "LG", + "ue_links": [], + "links_added_by_ue": [] + } + }, + { + "id": "f07d2d08-2bc5-4dd8-a9f0-f2347c6b5cca", + "version": 1, + "state": { + "lastGroupId": 3, + "lastNodeId": 134, + "lastLinkId": 235, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "LanPaint Preprocessing", + "inputNode": { + "id": -10, + "bounding": [ + 147.7819921827197, + 1983.6123610996979, + 120, + 100 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1453.72549451583, + 2003.6123610996979, + 120, + 100 + ] + }, + "inputs": [ + { + "id": "386e09d2-6c09-4494-8037-c52f3fc4d4fc", + "name": "vae", + "type": "VAE", + "linkIds": [ + 228, + 223, + 233 + ], + "localized_name": "vae", + "pos": [ + 247.7819921827197, + 2003.6123610996979 + ] + }, + { + "id": "686ea659-706d-48b5-aa90-ba8decf332ae", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 227, + 224 + ], + "localized_name": "image", + "pos": [ + 247.7819921827197, + 2023.6123610996979 + ] + }, + { + "id": "2f1a67f1-2cd1-46d9-8729-45755d6ca367", + "name": "mask", + "type": "MASK", + "linkIds": [ + 225 + ], + "localized_name": "mask", + "pos": [ + 247.7819921827197, + 2043.6123610996979 + ] + } + ], + "outputs": [ + { + "id": "c0e68b11-07aa-4ef5-bd82-9e26113be414", + "name": "LATENT", + "type": "LATENT", + "linkIds": [ + 226 + ], + "localized_name": "LATENT", + "pos": [ + 1473.72549451583, + 2023.6123610996979 + ] + }, + { + "id": "59d15e04-ac41-4f42-b0a0-1d4151cfe98b", + "name": "MASK", + "type": "MASK", + "linkIds": [ + 234 + ], + "pos": [ + 1473.72549451583, + 2043.6123610996979 + ] + }, + { + "id": "d597296a-7306-40e4-af12-04a7c2acc51f", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 235 + ], + "pos": [ + 1473.72549451583, + 2063.612361099698 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 125, + "type": "VAEDecode", + "pos": [ + 341.948202749188, + 1853.879018702556 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 212 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 228 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 213 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.3.71", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 126, + "type": "GetImageSize", + "pos": [ + 349.8229419100825, + 1980.2723036675939 + ], + "size": [ + 140, + 124 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 213 + } + ], + "outputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "links": [ + 215, + 218 + ] + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "links": [ + 216, + 219 + ] + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.7.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 127, + "type": "ImageScale", + "pos": [ + 705.0907645817256, + 2249.5210154889633 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 214 + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 215 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 216 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 217 + ] + } + ], + "properties": { + "Node name for S&R": "ImageScale", + "cnr_id": "comfy-core", + "ver": "0.3.52", + "ue_properties": { + "widget_ue_connectable": { + "width": true, + "height": true + }, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "nearest-exact", + 512, + 512, + "center" + ] + }, + { + "id": 124, + "type": "VAEEncode", + "pos": [ + 334.5563505623274, + 1685.8414078894357 + ], + "size": [ + 140, + 46 + ], + "flags": { + "collapsed": false + }, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "pixels", + "name": "pixels", + "type": "IMAGE", + "link": 224 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 223 + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 212 + ] + } + ], + "properties": { + "Node name for S&R": "VAEEncode", + "cnr_id": "comfy-core", + "ver": "0.3.71", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 129, + "type": "MaskToImage", + "pos": [ + 327.7819921827197, + 1780.2436686425217 + ], + "size": [ + 193.2779296875, + 26 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 225 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 214 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage", + "cnr_id": "comfy-core", + "ver": "0.3.51", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 132, + "type": "SetLatentNoiseMask", + "pos": [ + 1205.5881898283299, + 1723.8113483645336 + ], + "size": [ + 188.1373046875, + 46 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 232 + }, + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 222 + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 226 + ] + } + ], + "properties": { + "Node name for S&R": "SetLatentNoiseMask", + "cnr_id": "comfy-core", + "ver": "0.3.51", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 134, + "type": "VAEEncode", + "pos": [ + 1002.2749668743055, + 1594.9125557215868 + ], + "size": [ + 140, + 46 + ], + "flags": { + "collapsed": false + }, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "pixels", + "name": "pixels", + "type": "IMAGE", + "link": 231 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 233 + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 232 + ] + } + ], + "properties": { + "Node name for S&R": "VAEEncode", + "cnr_id": "comfy-core", + "ver": "0.3.71", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 128, + "type": "ImageToMask", + "pos": [ + 707.6856669146396, + 2444.3121664778087 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 217 + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 222, + 234 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToMask", + "cnr_id": "comfy-core", + "ver": "0.3.51", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "red" + ] + }, + { + "id": 130, + "type": "ImageScale", + "pos": [ + 722.567175008152, + 1753.1318522245451 + ], + "size": [ + 269.921875, + 152.8125 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 227 + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 218 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 219 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 231, + 235 + ] + } + ], + "properties": { + "Node name for S&R": "ImageScale", + "cnr_id": "comfy-core", + "ver": "0.3.52", + "ue_properties": { + "widget_ue_connectable": { + "width": true, + "height": true + }, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "area", + 512, + 512, + "center" + ] + } + ], + "groups": [], + "links": [ + { + "id": 212, + "origin_id": 124, + "origin_slot": 0, + "target_id": 125, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 213, + "origin_id": 125, + "origin_slot": 0, + "target_id": 126, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 214, + "origin_id": 129, + "origin_slot": 0, + "target_id": 127, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 215, + "origin_id": 126, + "origin_slot": 0, + "target_id": 127, + "target_slot": 1, + "type": "INT" + }, + { + "id": 216, + "origin_id": 126, + "origin_slot": 1, + "target_id": 127, + "target_slot": 2, + "type": "INT" + }, + { + "id": 217, + "origin_id": 127, + "origin_slot": 0, + "target_id": 128, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 218, + "origin_id": 126, + "origin_slot": 0, + "target_id": 130, + "target_slot": 1, + "type": "INT" + }, + { + "id": 219, + "origin_id": 126, + "origin_slot": 1, + "target_id": 130, + "target_slot": 2, + "type": "INT" + }, + { + "id": 232, + "origin_id": 134, + "origin_slot": 0, + "target_id": 132, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 222, + "origin_id": 128, + "origin_slot": 0, + "target_id": 132, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 231, + "origin_id": 130, + "origin_slot": 0, + "target_id": 134, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 228, + "origin_id": -10, + "origin_slot": 0, + "target_id": 125, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 223, + "origin_id": -10, + "origin_slot": 0, + "target_id": 124, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 233, + "origin_id": -10, + "origin_slot": 0, + "target_id": 134, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 227, + "origin_id": -10, + "origin_slot": 1, + "target_id": 130, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 224, + "origin_id": -10, + "origin_slot": 1, + "target_id": 124, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 225, + "origin_id": -10, + "origin_slot": 2, + "target_id": 129, + "target_slot": 0, + "type": "MASK" + }, + { + "id": 226, + "origin_id": 132, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 234, + "origin_id": 128, + "origin_slot": 0, + "target_id": -20, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 235, + "origin_id": 130, + "origin_slot": 0, + "target_id": -20, + "target_slot": 2, + "type": "IMAGE" + } + ], + "extra": { + "ue_links": [], + "links_added_by_ue": [], + "workflowRendererVersion": "LG" + } + } + ] + }, + "config": {}, + "extra": { + "frontendVersion": "1.37.11", + "workflowRendererVersion": "LG", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "ue_links": [], + "links_added_by_ue": [], + "ds": { + "scale": 0.6934334949441376, + "offset": [ + -453.34680325511, + -421.81393627582696 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Flux_Inpaint.jpg b/LanPaint/example_workflows/Flux_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1c099094d3b4de9b4a90de49d9624349a2ad5420 --- /dev/null +++ b/LanPaint/example_workflows/Flux_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e826efb4e7dad79b01605a9a77700ec4209d14825526405e67703aa79a7719c +size 1337757 diff --git a/LanPaint/example_workflows/Flux_Inpaint.json b/LanPaint/example_workflows/Flux_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..9169e23644116adcbc58489108b81bb2cf45acab --- /dev/null +++ b/LanPaint/example_workflows/Flux_Inpaint.json @@ -0,0 +1,786 @@ +{ + "id": "978d3a45-3d13-43c6-8ef9-89dc3e74d6ba", + "revision": 0, + "last_node_id": 82, + "last_link_id": 220, + "nodes": [ + { + "id": 66, + "type": "SetLatentNoiseMask", + "pos": [ + 514.1915893554688, + 781.9396362304688 + ], + "size": [ + 264.5999755859375, + 46 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 216 + }, + { + "name": "mask", + "type": "MASK", + "link": 189 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 186 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 78, + "type": "CLIPTextEncode", + "pos": [ + 314.8565368652344, + 255.63235473632812 + ], + "size": [ + 422.84503173828125, + 164.31304931640625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "label": "clip", + "name": "clip", + "type": "CLIP", + "link": 194 + } + ], + "outputs": [ + { + "label": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 195 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "cute anime girl with massive fluffy fennec ears and a big fluffy tail blonde messy long hair blue eyes wearing a maid outfit with a long black gold leaf pattern dress and a white apron mouth open placing a fancy black forest cake with candles on top of a dinner table of an old dark Victorian mansion lit by candlelight with a bright window to the foggy forest and very expensive stuff everywhere there are paintings on the walls" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 80, + "type": "CLIPTextEncode", + "pos": [ + 362.7684020996094, + 481.0662536621094 + ], + "size": [ + 422.84503173828125, + 164.31304931640625 + ], + "flags": { + "collapsed": true + }, + "order": 3, + "mode": 0, + "inputs": [ + { + "label": "clip", + "name": "clip", + "type": "CLIP", + "link": 196 + } + ], + "outputs": [ + { + "label": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 210 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 79, + "type": "FluxGuidance", + "pos": [ + 529.1380615234375, + 156.20236206054688 + ], + "size": [ + 211.60000610351562, + 58 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "label": "conditioning", + "name": "conditioning", + "type": "CONDITIONING", + "link": 195 + } + ], + "outputs": [ + { + "label": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 207 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "FluxGuidance" + }, + "widgets_values": [ + 3.5 + ] + }, + { + "id": 77, + "type": "CheckpointLoaderSimple", + "pos": [ + -66.25785064697266, + 484.081787109375 + ], + "size": [ + 315, + 98 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "MODEL", + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 197 + ] + }, + { + "label": "CLIP", + "name": "CLIP", + "type": "CLIP", + "slot_index": 1, + "links": [ + 194, + 196 + ] + }, + { + "label": "VAE", + "name": "VAE", + "type": "VAE", + "slot_index": 2, + "links": [ + 200, + 203 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "flux1-dev-fp8.safetensors" + ] + }, + { + "id": 73, + "type": "LanPaint_KSampler", + "pos": [ + 1005.0399780273438, + 296.9553527832031 + ], + "size": [ + 413.6495666503906, + 596 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 197 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 207 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 210 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 186 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 187 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "56bd6c04e89124cd06682b304245d6ddf8b20522", + "Node name for S&R": "LanPaint_KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 1, + "euler", + "simple", + 1, + 5, + "Image First", + "LanPaint KSampler. Recommend steps 50, LanPaint NumSteps 1-20 depending on the difficulty of task. LanPaint_EndSigma = 3.0 for anime style, 0.6 for realistic style. For more information, visit https://github.com/scraed/LanPaint", + "🖼️ Image Inpainting" + ] + }, + { + "id": 65, + "type": "VAEEncode", + "pos": [ + 200.1034698486328, + 776.3561401367188 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 188 + }, + { + "name": "vae", + "type": "VAE", + "link": 203 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 216 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [] + }, + { + "id": 48, + "type": "SaveImage", + "pos": [ + 1075.49169921875, + 1167.2703857421875 + ], + "size": [ + 311.2532653808594, + 484.7096252441406 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 103 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 81, + "type": "LanPaint_MaskBlend", + "pos": [ + 1773.0189208984375, + 1222.9478759765625 + ], + "size": [ + 210, + 98 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 219 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 218 + }, + { + "name": "mask", + "type": "MASK", + "link": 220 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 217 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712", + "Node name for S&R": "LanPaint_MaskBlend" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 82, + "type": "SaveImage", + "pos": [ + 2106.45166015625, + 1090.682861328125 + ], + "size": [ + 311.2532653808594, + 484.7096252441406 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 217 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1211.46484375, + 1065.318359375 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 187 + }, + { + "name": "vae", + "type": "VAE", + "link": 200 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 103, + 218 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 75, + "type": "LoadImage", + "pos": [ + 45.954593658447266, + 1150.45556640625 + ], + "size": [ + 266.13720703125, + 487.1314697265625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 188, + 219 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 189, + 220 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "clipspace/clipspace-mask-8865503.5.png [input]", + "image" + ] + } + ], + "links": [ + [ + 103, + 8, + 0, + 48, + 0, + "IMAGE" + ], + [ + 186, + 66, + 0, + 73, + 3, + "LATENT" + ], + [ + 187, + 73, + 0, + 8, + 0, + "LATENT" + ], + [ + 188, + 75, + 0, + 65, + 0, + "IMAGE" + ], + [ + 189, + 75, + 1, + 66, + 1, + "MASK" + ], + [ + 194, + 77, + 1, + 78, + 0, + "CLIP" + ], + [ + 195, + 78, + 0, + 79, + 0, + "CONDITIONING" + ], + [ + 196, + 77, + 1, + 80, + 0, + "CLIP" + ], + [ + 197, + 77, + 0, + 73, + 0, + "MODEL" + ], + [ + 200, + 77, + 2, + 8, + 1, + "VAE" + ], + [ + 203, + 77, + 2, + 65, + 1, + "VAE" + ], + [ + 207, + 79, + 0, + 73, + 1, + "CONDITIONING" + ], + [ + 210, + 80, + 0, + 73, + 2, + "CONDITIONING" + ], + [ + 216, + 65, + 0, + 66, + 0, + "LATENT" + ], + [ + 217, + 81, + 0, + 82, + 0, + "IMAGE" + ], + [ + 218, + 8, + 0, + 81, + 1, + "IMAGE" + ], + [ + 219, + 75, + 0, + 81, + 0, + "IMAGE" + ], + [ + 220, + 75, + 1, + 81, + 2, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "Mask image for inpainting.", + "bounding": [ + 36.04227828979492, + 989.7313232421875, + 278.89093017578125, + 669.3414916992188 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Convert Latents for LanPaint", + "bounding": [ + 286.0640563964844, + 714.343505859375, + 489.16796875, + 197.81044006347656 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model and Set Prompts", + "bounding": [ + -78.9311294555664, + 176.08712768554688, + 1006.1721801757812, + 514.258544921875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Inpaint with the LanPaint KSampler", + "bounding": [ + 960.8922729492188, + 179.17588806152344, + 474.8909606933594, + 630.4742431640625 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "LanPaint OutPut", + "bounding": [ + 1085.6029052734375, + 994.0775756835938, + 345.4561767578125, + 669.4969482421875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 11, + "title": "LanPaint", + "bounding": [ + -262.59381103515625, + 140.46656799316406, + 1737.328857421875, + 797.4443359375 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.35049389948139237, + "offset": [ + 348.866804381099, + 308.65057628971834 + ] + }, + "frontendVersion": "1.27.10", + "node_versions": { + "comfy-core": "0.3.18", + "LanPaint": "0f509469ed2cd60c6032f739e282aad5dfc06166" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Hunyuan_Inpaint.jpg b/LanPaint/example_workflows/Hunyuan_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4462ca003c46b79cfa0f330c210412621c56b102 --- /dev/null +++ b/LanPaint/example_workflows/Hunyuan_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd0114520644c017e18913b10c70c24b16c3366e4449035293e2b9f0fe1a56b2 +size 685947 diff --git a/LanPaint/example_workflows/Hunyuan_Inpaint.json b/LanPaint/example_workflows/Hunyuan_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..a659601cb54765dca3616a1965f72a219e8aa7ee --- /dev/null +++ b/LanPaint/example_workflows/Hunyuan_Inpaint.json @@ -0,0 +1,1033 @@ +{ + "id": "054799b4-76bf-4b4e-b7b7-615d2914285b", + "revision": 0, + "last_node_id": 117, + "last_link_id": 265, + "nodes": [ + { + "id": 88, + "type": "Note", + "pos": [ + 180, + 440 + ], + "size": [ + 350, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Select a fp8 weight_dtype if you are running out of memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 103, + "type": "UNETLoader", + "pos": [ + 180, + 310 + ], + "size": [ + 350, + 82 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 241, + 242, + 250 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "UNETLoader", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "hunyuan_video_t2v_720p_bf16.safetensors", + "default" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 94, + "type": "DualCLIPLoader", + "pos": [ + -160, + -60 + ], + "size": [ + 350, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 244, + 252, + 255 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "DualCLIPLoader", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "clip_l.safetensors", + "llava_llama3_fp8_scaled.safetensors", + "hunyuan_video", + "default" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 114, + "type": "FluxGuidance", + "pos": [ + 653.5101318359375, + -73.20354461669922 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 258 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 259 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.65", + "Node name for S&R": "FluxGuidance" + }, + "widgets_values": [ + 6 + ] + }, + { + "id": 106, + "type": "VAEEncode", + "pos": [ + 558.60888671875, + 922.0687255859375 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 216 + }, + { + "name": "vae", + "type": "VAE", + "link": 220 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 217 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "VAEEncode", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 107, + "type": "SetLatentNoiseMask", + "pos": [ + 732.1127319335938, + 906.8209228515625 + ], + "size": [ + 180.74765014648438, + 46 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 217 + }, + { + "name": "mask", + "type": "MASK", + "link": 218 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 261 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "SetLatentNoiseMask", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 105, + "type": "LoadImage", + "pos": [ + 171.56072998046875, + 795.9495239257812 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 216 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 218 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "LoadImage", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Masked_Load_Me_in_Loader (8).png", + "image" + ] + }, + { + "id": 104, + "type": "CLIPTextEncode", + "pos": [ + 226.1235809326172, + -161.8229522705078 + ], + "size": [ + 310, + 240 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 252 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 245, + 258 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "CLIPTextEncode", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Close up Portrait of a futuristic cowgirl wearing jeans rides a mechanical horse across a vast cyberpunk desert. The dunes shimmer with holographic projections, glowing in blues and purples. cacti and two massive moons hang low on the horizon, illuminating the dusty air. Cyberpunk western fusion, cinematic, 8K, highly detailed, atmospheric perspective, Blade Runner style." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 113, + "type": "CLIPTextEncode", + "pos": [ + 233.61778259277344, + 122.75636291503906 + ], + "size": [ + 319.00726318359375, + 88 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 255 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 254 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "CLIPTextEncode", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 92, + "type": "Note", + "pos": [ + 1340, + 250 + ], + "size": [ + 210, + 170 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Use the tiled decode node by default because most people will need it.\n\nLower the tile_size and overlap if you run out of memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 116, + "type": "Note", + "pos": [ + 1324.0262451171875, + 583.7811889648438 + ], + "size": [ + 210, + 170 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "Decrease LanPaint_NumSteps to accelerate" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VAEDecodeTiled", + "pos": [ + 1341.9100341796875, + 49.04500961303711 + ], + "size": [ + 210, + 150 + ], + "flags": {}, + "order": 14, + "mode": 4, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 248 + }, + { + "name": "vae", + "type": "VAE", + "link": 237 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 246, + 247, + 251 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.18", + "Node name for S&R": "VAEDecodeTiled", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 256, + 64, + 64, + 8 + ] + }, + { + "id": 86, + "type": "SaveAnimatedWEBP", + "pos": [ + 1661.5201416015625, + 68.0674819946289 + ], + "size": [ + 380, + 366 + ], + "flags": {}, + "order": 16, + "mode": 4, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 251 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.18", + "Node name for S&R": "SaveAnimatedWEBP", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "ComfyUI", + 24, + false, + 80, + "default" + ] + }, + { + "id": 110, + "type": "LanPaint_KSampler", + "pos": [ + 829.425537109375, + 218.98995971679688 + ], + "size": [ + 400, + 596 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 239 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 259 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 254 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 261 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 248, + 262 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "6109df6591a4cf2bc9d3b113d03f7297fa9248e9", + "Node name for S&R": "LanPaint_KSampler" + }, + "widgets_values": [ + 534861079790570, + "randomize", + 20, + 1, + "euler", + "simple", + 1, + 5, + "Image First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", + "🖼️ Image Inpainting" + ] + }, + { + "id": 89, + "type": "VAEDecode", + "pos": [ + 1340, + -60 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 262 + }, + { + "name": "vae", + "type": "VAE", + "link": 265 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 264 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.18", + "Node name for S&R": "VAEDecode", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 78, + "type": "VAELoader", + "pos": [ + 910, + -230 + ], + "size": [ + 350.65509033203125, + 58 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 229, + 232, + 220, + 237, + 265 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.18", + "Node name for S&R": "VAELoader", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "hunyuan_video_vae_bf16.safetensors" + ] + }, + { + "id": 81, + "type": "ModelSamplingSD3", + "pos": [ + 613.9942626953125, + 59.9338493347168 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 250 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 226, + 239 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.18", + "Node name for S&R": "ModelSamplingSD3", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 7 + ] + }, + { + "id": 117, + "type": "SaveImage", + "pos": [ + 1670.939697265625, + -104.6227798461914 + ], + "size": [ + 531.0060424804688, + 550.45361328125 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 264 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.65" + }, + "widgets_values": [ + "ComfyUI" + ] + } + ], + "links": [ + [ + 216, + 105, + 0, + 106, + 0, + "IMAGE" + ], + [ + 217, + 106, + 0, + 107, + 0, + "LATENT" + ], + [ + 218, + 105, + 1, + 107, + 1, + "MASK" + ], + [ + 220, + 78, + 0, + 106, + 1, + "VAE" + ], + [ + 237, + 78, + 0, + 85, + 1, + "VAE" + ], + [ + 239, + 81, + 0, + 110, + 0, + "MODEL" + ], + [ + 248, + 110, + 0, + 85, + 0, + "LATENT" + ], + [ + 250, + 103, + 0, + 81, + 0, + "MODEL" + ], + [ + 251, + 85, + 0, + 86, + 0, + "IMAGE" + ], + [ + 252, + 94, + 0, + 104, + 0, + "CLIP" + ], + [ + 254, + 113, + 0, + 110, + 2, + "CONDITIONING" + ], + [ + 255, + 94, + 0, + 113, + 0, + "CLIP" + ], + [ + 258, + 104, + 0, + 114, + 0, + "CONDITIONING" + ], + [ + 259, + 114, + 0, + 110, + 1, + "CONDITIONING" + ], + [ + 261, + 107, + 0, + 110, + 3, + "LATENT" + ], + [ + 262, + 110, + 0, + 89, + 0, + "LATENT" + ], + [ + 264, + 89, + 0, + 117, + 0, + "IMAGE" + ], + [ + 265, + 78, + 0, + 89, + 1, + "VAE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Text Encoding", + "bounding": [ + -190, + -140, + 730, + 340 + ], + "color": "#8A8", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "VAE Decoding & Output", + "bounding": [ + 1320, + -140, + 1110, + 600 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Sampling Process", + "bounding": [ + 570, + -140, + 730, + 980 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "HunyuanVideo Model", + "bounding": [ + 170, + 230, + 370.5005798339844, + 330.7360534667969 + ], + "color": "#88A", + "font_size": 24, + "flags": {} + }, + { + "id": 6, + "title": "Guidance", + "bounding": [ + 590, + -90, + 680, + 260 + ], + "color": "#8AA", + "font_size": 24, + "flags": {} + }, + { + "id": 7, + "title": "sampling", + "bounding": [ + 590, + 200, + 680, + 440 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.4870776476432723, + "offset": [ + 85.03251058761712, + 779.6524460150363 + ] + }, + "frontendVersion": "1.27.10", + "groupNodes": {}, + "ue_links": [], + "links_added_by_ue": [], + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Masked_Qwen_Image_Edit.jpg b/LanPaint/example_workflows/Masked_Qwen_Image_Edit.jpg new file mode 100644 index 0000000000000000000000000000000000000000..77e085b2878afd9a9ae3b088470ab4fe9c0ab7df --- /dev/null +++ b/LanPaint/example_workflows/Masked_Qwen_Image_Edit.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9994fe2e38218474e755959c00ab081c7e90565746132cba359da98b57240a76 +size 689931 diff --git a/LanPaint/example_workflows/Masked_Qwen_Image_Edit.json b/LanPaint/example_workflows/Masked_Qwen_Image_Edit.json new file mode 100644 index 0000000000000000000000000000000000000000..213b4027b544c1f9e5d53e7280bd4655b4365384 --- /dev/null +++ b/LanPaint/example_workflows/Masked_Qwen_Image_Edit.json @@ -0,0 +1,1636 @@ +{ + "id": "91f6bbe2-ed41-4fd6-bac7-71d5b5864ecb", + "revision": 0, + "last_node_id": 147, + "last_link_id": 315, + "nodes": [ + { + "id": 99, + "type": "MarkdownNote", + "pos": [ + -830, + -10 + ], + "size": [ + 540, + 550 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "[Tutorial](https://docs.comfy.org/tutorials/image/qwen/qwen-image-edit) | [教程](https://docs.comfy.org/zh-CN/tutorials/image/qwen/qwen-image-edit)\n\n\n## Model links\n\nYou can find all the models on [Comfy-Org/Qwen-Image_ComfyUI](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main) and [Comfy-Org/Qwen-Image-Edit_ComfyUI](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI) \n\n**Diffusion model**\n\n- [qwen_image_edit_fp8_e4m3fn.safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/resolve/main/split_files/diffusion_models/qwen_image_edit_fp8_e4m3fn.safetensors)\n\n**LoRA**\n\n- [Qwen-Image-Lightning-4steps-V1.0.safetensors](https://huggingface.co/lightx2v/Qwen-Image-Lightning/resolve/main/Qwen-Image-Lightning-4steps-V1.0.safetensors)\n\n**Text encoder**\n\n- [qwen_2.5_vl_7b_fp8_scaled.safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors)\n\n**VAE**\n\n- [qwen_image_vae.safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/qwen_image_vae.safetensors)\n\nModel Storage Location\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 diffusion_models/\n│ │ └── qwen_image_edit_fp8_e4m3fn.safetensors\n│ ├── 📂 loras/\n│ │ └── Qwen-Image-Lightning-4steps-V1.0.safetensors\n│ ├── 📂 vae/\n│ │ └── qwen_image_vae.safetensors\n│ └── 📂 text_encoders/\n│ └── qwen_2.5_vl_7b_fp8_scaled.safetensors\n```\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 129, + "type": "ModelSamplingAuraFlow", + "pos": [ + 550.1956176757812, + 27.127967834472656 + ], + "size": [ + 290, + 60 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 293 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 260 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "ModelSamplingAuraFlow", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 130, + "type": "CFGNorm", + "pos": [ + 550.1956176757812, + 137.1279754638672 + ], + "size": [ + 290, + 60 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 260 + } + ], + "outputs": [ + { + "name": "patched_model", + "type": "MODEL", + "links": [ + 269 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "CFGNorm", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "strength": true + } + } + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 138, + "type": "LanPaint_MaskBlend", + "pos": [ + 1746.5638427734375, + 60.70713806152344 + ], + "size": [ + 210, + 98 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 309 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 286 + }, + { + "name": "mask", + "type": "MASK", + "link": 290 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 291 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712", + "Node name for S&R": "LanPaint_MaskBlend" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 97, + "type": "MarkdownNote", + "pos": [ + 960.13525390625, + 776.3380737304688 + ], + "size": [ + 300, + 190 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "KSampler settings", + "properties": {}, + "widgets_values": [ + "You can test and find the best setting by yourself. The following table is for reference.\n\n| Model | Steps | CFG |\n|---------------------|---------------|---------------|\n| Offical | 50 | 4.0 \n| fp8_e4m3fn | 20 | 2.5 |\n| fp8_e4m3fn + 4steps LoRA | 4 | 1.0 |\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 124, + "type": "UNETLoader", + "pos": [ + -249.80447387695312, + 37.12797164916992 + ], + "size": [ + 330, + 90 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 292 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "qwen_image_edit_fp8_e4m3fn.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/resolve/main/split_files/diffusion_models/qwen_image_edit_fp8_e4m3fn.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_edit_fp8_e4m3fn.safetensors", + "default" + ] + }, + { + "id": 141, + "type": "LoraLoaderModelOnly", + "pos": [ + 199.41346740722656, + 19.401098251342773 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 8, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 292 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 293 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoraLoaderModelOnly", + "models": [ + { + "name": "Qwen-Image-Lightning-4steps-V1.0.safetensors", + "url": "https://huggingface.co/lightx2v/Qwen-Image-Lightning/resolve/main/Qwen-Image-Lightning-4steps-V1.0.safetensors", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "lora_name": true, + "strength_model": true + } + } + }, + "widgets_values": [ + "Qwen-Image-Lightning-4steps-V1.0.safetensors", + 1 + ] + }, + { + "id": 125, + "type": "CLIPLoader", + "pos": [ + -249.80447387695312, + 177.12796020507812 + ], + "size": [ + 330, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 255, + 257 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "qwen_image", + "default" + ] + }, + { + "id": 104, + "type": "SaveImage", + "pos": [ + 2063.178955078125, + 10.873106002807617 + ], + "size": [ + 311.2532653808594, + 484.7096252441406 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 291 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 127, + "type": "TextEncodeQwenImageEdit", + "pos": [ + 182.30767822265625, + 198.8886260986328 + ], + "size": [ + 360, + 150 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 255 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 256 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 306 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 267 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "TextEncodeQwenImageEdit", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "prompt": true + } + } + }, + "widgets_values": [ + "change the girl's cloth to red evening gown " + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 128, + "type": "TextEncodeQwenImageEdit", + "pos": [ + 190.8400421142578, + 395.84197998046875 + ], + "size": [ + 360, + 150 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 257 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 258 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 307 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 268 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "TextEncodeQwenImageEdit", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "prompt": true + } + } + }, + "widgets_values": [ + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 134, + "type": "SetLatentNoiseMask", + "pos": [ + 560.0880126953125, + 642.5905151367188 + ], + "size": [ + 180.74765014648438, + 46 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 312 + }, + { + "name": "mask", + "type": "MASK", + "link": 273 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 284 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 147, + "type": "VAEEncode", + "pos": [ + 381.33148193359375, + 639.6804809570312 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 311 + }, + { + "name": "vae", + "type": "VAE", + "link": 313 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 312 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "VAEEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {} + } + } + }, + { + "id": 126, + "type": "VAELoader", + "pos": [ + -249.80447387695312, + 337.1279296875 + ], + "size": [ + 330, + 60 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 256, + 258, + 276, + 288, + 296, + 313 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "qwen_image_vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/qwen_image_vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_vae.safetensors" + ] + }, + { + "id": 142, + "type": "VAEDecode", + "pos": [ + -468.50421142578125, + 1343.047119140625 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 294 + }, + { + "name": "vae", + "type": "VAE", + "link": 296 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 297 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 144, + "type": "ImageScale", + "pos": [ + -163.11903381347656, + 1055.310546875 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 314 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 298 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 299 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 306, + 307, + 309, + 311 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "ImageScale" + }, + "widgets_values": [ + "nearest-exact", + 512, + 512, + "center" + ] + }, + { + "id": 110, + "type": "LoadImage", + "pos": [ + -224.69882202148438, + 491.67425537109375 + ], + "size": [ + 274.080078125, + 314.0000305175781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 275, + 314 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 277 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoadImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "image": true, + "upload": true + } + } + }, + "widgets_values": [ + "clipspace/clipspace-painted-masked-23308506.png [input]", + "image" + ] + }, + { + "id": 136, + "type": "MaskToImage", + "pos": [ + -121.25953674316406, + 1228.22509765625 + ], + "size": [ + 184.62362670898438, + 26 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 277 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 315 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "MaskToImage" + }, + "widgets_values": [] + }, + { + "id": 133, + "type": "ImageToMask", + "pos": [ + -164.2398681640625, + 1480.3736572265625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 305 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 273, + 290 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "ImageToMask" + }, + "widgets_values": [ + "red" + ] + }, + { + "id": 96, + "type": "MarkdownNote", + "pos": [ + -548.3401489257812, + 1051.436279296875 + ], + "size": [ + 280, + 88 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "This node is to avoid poor output results caused by excessively large input image sizes." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 135, + "type": "VAEEncode", + "pos": [ + -453.77728271484375, + 1250.4849853515625 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 310 + }, + { + "name": "vae", + "type": "VAE", + "link": 276 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 294 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "VAEEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 146, + "type": "ImageScale", + "pos": [ + -165.53955078125, + 1306.8914794921875 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 315 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 301 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 302 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 305 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "ImageScale" + }, + "widgets_values": [ + "nearest-exact", + 512, + 512, + "center" + ] + }, + { + "id": 140, + "type": "MarkdownNote", + "pos": [ + 1361.85009765625, + 360.32342529296875 + ], + "size": [ + 300, + 190 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "KSampler settings", + "properties": {}, + "widgets_values": [ + "Decrease **LanPaint_NumSteps** for faster generation. \n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 145, + "type": "GetImageSize", + "pos": [ + -456.3453063964844, + 1442.398193359375 + ], + "size": [ + 140, + 124 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 297 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 298, + 301 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 299, + 302 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "GetImageSize" + }, + "widgets_values": [ + "width: 1024, height: 1024\n batch size: 1" + ] + }, + { + "id": 131, + "type": "ImageScaleToTotalPixels", + "pos": [ + -851.1531982421875, + 1230.9898681640625 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 275 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 310 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "ImageScaleToTotalPixels", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "upscale_method": true, + "megapixels": true + } + } + }, + "widgets_values": [ + "nearest-exact", + 1 + ] + }, + { + "id": 102, + "type": "LanPaint_KSampler", + "pos": [ + 935.6971435546875, + 22.220111846923828 + ], + "size": [ + 400, + 572 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 269 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 267 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 268 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 284 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 287 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "453b6c090e400123bb82f5bc0c97854031b09274", + "Node name for S&R": "LanPaint_KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 20, + 2.5, + "euler", + "simple", + 1, + 5, + "Image First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + }, + { + "id": 137, + "type": "VAEDecode", + "pos": [ + 1453.6231689453125, + 22.1680850982666 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 287 + }, + { + "name": "vae", + "type": "VAE", + "link": 288 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 286 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 255, + 125, + 0, + 127, + 0, + "CLIP" + ], + [ + 256, + 126, + 0, + 127, + 1, + "VAE" + ], + [ + 257, + 125, + 0, + 128, + 0, + "CLIP" + ], + [ + 258, + 126, + 0, + 128, + 1, + "VAE" + ], + [ + 260, + 129, + 0, + 130, + 0, + "MODEL" + ], + [ + 267, + 127, + 0, + 102, + 1, + "CONDITIONING" + ], + [ + 268, + 128, + 0, + 102, + 2, + "CONDITIONING" + ], + [ + 269, + 130, + 0, + 102, + 0, + "MODEL" + ], + [ + 273, + 133, + 0, + 134, + 1, + "MASK" + ], + [ + 275, + 110, + 0, + 131, + 0, + "IMAGE" + ], + [ + 276, + 126, + 0, + 135, + 1, + "VAE" + ], + [ + 277, + 110, + 1, + 136, + 0, + "MASK" + ], + [ + 284, + 134, + 0, + 102, + 3, + "LATENT" + ], + [ + 286, + 137, + 0, + 138, + 1, + "IMAGE" + ], + [ + 287, + 102, + 0, + 137, + 0, + "LATENT" + ], + [ + 288, + 126, + 0, + 137, + 1, + "VAE" + ], + [ + 290, + 133, + 0, + 138, + 2, + "MASK" + ], + [ + 291, + 138, + 0, + 104, + 0, + "IMAGE" + ], + [ + 292, + 124, + 0, + 141, + 0, + "MODEL" + ], + [ + 293, + 141, + 0, + 129, + 0, + "MODEL" + ], + [ + 294, + 135, + 0, + 142, + 0, + "LATENT" + ], + [ + 296, + 126, + 0, + 142, + 1, + "VAE" + ], + [ + 297, + 142, + 0, + 145, + 0, + "IMAGE" + ], + [ + 298, + 145, + 0, + 144, + 1, + "INT" + ], + [ + 299, + 145, + 1, + 144, + 2, + "INT" + ], + [ + 301, + 145, + 0, + 146, + 1, + "INT" + ], + [ + 302, + 145, + 1, + 146, + 2, + "INT" + ], + [ + 305, + 146, + 0, + 133, + 0, + "IMAGE" + ], + [ + 306, + 144, + 0, + 127, + 2, + "IMAGE" + ], + [ + 307, + 144, + 0, + 128, + 2, + "IMAGE" + ], + [ + 309, + 144, + 0, + 138, + 0, + "IMAGE" + ], + [ + 310, + 131, + 0, + 135, + 0, + "IMAGE" + ], + [ + 311, + 144, + 0, + 147, + 0, + "IMAGE" + ], + [ + 312, + 147, + 0, + 134, + 0, + "LATENT" + ], + [ + 313, + 126, + 0, + 147, + 1, + "VAE" + ], + [ + 314, + 110, + 0, + 144, + 0, + "IMAGE" + ], + [ + 315, + 136, + 0, + 146, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Step1 - Load models", + "bounding": [ + -270, + -40, + 370, + 450 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Step 2 - Upload image for editing", + "bounding": [ + -270, + 430, + 370, + 400 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Step 3 - Prompt", + "bounding": [ + 130, + 130, + 380, + 433.6000061035156 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8770472573513671, + "offset": [ + -1288.739882247765, + 215.50013247190662 + ] + }, + "frontendVersion": "1.25.10", + "ue_links": [], + "links_added_by_ue": [], + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "groupNodes": {} + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Masked_Qwen_Image_Edit_2509.jpg b/LanPaint/example_workflows/Masked_Qwen_Image_Edit_2509.jpg new file mode 100644 index 0000000000000000000000000000000000000000..77e085b2878afd9a9ae3b088470ab4fe9c0ab7df --- /dev/null +++ b/LanPaint/example_workflows/Masked_Qwen_Image_Edit_2509.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9994fe2e38218474e755959c00ab081c7e90565746132cba359da98b57240a76 +size 689931 diff --git a/LanPaint/example_workflows/Masked_Qwen_Image_Edit_2509.json b/LanPaint/example_workflows/Masked_Qwen_Image_Edit_2509.json new file mode 100644 index 0000000000000000000000000000000000000000..41e691550264f0a5bf6dd6edc528f303559c3ece --- /dev/null +++ b/LanPaint/example_workflows/Masked_Qwen_Image_Edit_2509.json @@ -0,0 +1,1829 @@ +{ + "id": "91f6bbe2-ed41-4fd6-bac7-71d5b5864ecb", + "revision": 0, + "last_node_id": 127, + "last_link_id": 252, + "nodes": [ + { + "id": 38, + "type": "CLIPLoader", + "pos": [ + -250, + 160 + ], + "size": [ + 330, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 204, + 205 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "qwen_image", + "default" + ] + }, + { + "id": 66, + "type": "ModelSamplingAuraFlow", + "pos": [ + 740, + -130 + ], + "size": [ + 290, + 60 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 185 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 141 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "ModelSamplingAuraFlow", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 106, + "type": "LoadImage", + "pos": [ + 56.080078125, + 530 + ], + "size": [ + 274.080078125, + 314.0000305175781 + ], + "flags": {}, + "order": 1, + "mode": 4, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 219, + 220 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoadImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "image": true, + "upload": true + } + } + }, + "widgets_values": [ + "0_1 (5).png", + "image" + ] + }, + { + "id": 108, + "type": "LoadImage", + "pos": [ + 352.16015625, + 530 + ], + "size": [ + 274.080078125, + 314.0000305175781 + ], + "flags": {}, + "order": 2, + "mode": 4, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 217, + 218 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoadImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "image": true, + "upload": true + } + } + }, + "widgets_values": [ + "pose.png", + "image" + ] + }, + { + "id": 112, + "type": "EmptySD3LatentImage", + "pos": [ + 750, + 860 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "EmptySD3LatentImage" + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 113, + "type": "MarkdownNote", + "pos": [ + 730, + 1030 + ], + "size": [ + 330, + 90 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note: About image size", + "properties": {}, + "widgets_values": [ + "You can use the latent from the **EmptySD3LatentImage** to replace **VAE Encode**, so you can customize the image size." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 97, + "type": "MarkdownNote", + "pos": [ + 740, + 610 + ], + "size": [ + 300, + 160 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note: KSampler settings", + "properties": {}, + "widgets_values": [ + "You can test and find the best setting by yourself. The following table is for reference.\n\n| Model | Steps | CFG |\n|---------------------|---------------|---------------|\n| Offical | 50 | 4.0 \n| fp8_e4m3fn | 20 | 2.5 |\n| fp8_e4m3fn + 4steps LoRA | 4 | 1.0 |\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 110, + "type": "TextEncodeQwenImageEditPlus", + "pos": [ + 220.83323669433594, + 178.33360290527344 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 204 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 206 + }, + { + "name": "image1", + "shape": 7, + "type": "IMAGE", + "link": 225 + }, + { + "name": "image2", + "shape": 7, + "type": "IMAGE", + "link": 220 + }, + { + "name": "image3", + "shape": 7, + "type": "IMAGE", + "link": 218 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 245 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "TextEncodeQwenImageEditPlus" + }, + "widgets_values": [ + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 99, + "type": "MarkdownNote", + "pos": [ + -840, + -140 + ], + "size": [ + 550, + 550 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "[Tutorial](https://docs.comfy.org/tutorials/image/qwen/qwen-image-edit) | [教程](https://docs.comfy.org/zh-CN/tutorials/image/qwen/qwen-image-edit)\n\n\n## Model links\n\nYou can find all the models on [Comfy-Org/Qwen-Image_ComfyUI](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main) and [Comfy-Org/Qwen-Image-Edit_ComfyUI](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI) \n\n**Diffusion model**\n\n- [qwen_image_edit_2509_fp8_e4m3fn.safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/resolve/main/split_files/diffusion_models/qwen_image_edit_2509_fp8_e4m3fn.safetensors)\n\n**LoRA**\n\n- [Qwen-Image-Lightning-4steps-V1.0.safetensors](https://huggingface.co/lightx2v/Qwen-Image-Lightning/resolve/main/Qwen-Image-Lightning-4steps-V1.0.safetensors)\n\n**Text encoder**\n\n- [qwen_2.5_vl_7b_fp8_scaled.safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors)\n\n**VAE**\n\n- [qwen_image_vae.safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/qwen_image_vae.safetensors)\n\nModel Storage Location\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 diffusion_models/\n│ │ └── qwen_image_edit_2509_fp8_e4m3fn.safetensors\n│ ├── 📂 loras/\n│ │ └── Qwen-Image-Lightning-4steps-V1.0.safetensors\n│ ├── 📂 vae/\n│ │ └── qwen_image_vae.safetensors\n│ └── 📂 text_encoders/\n│ └── qwen_2.5_vl_7b_fp8_scaled.safetensors\n```\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "MarkdownNote", + "pos": [ + -240, + 1030 + ], + "size": [ + 290, + 140 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "This node is to avoid bad output results caused by excessively large input image sizes. Because when we input one image, we use the size of that input image.\n\nThe **TextEncodeQwenImageEditPlus** will scale your input to 1024×104 pixels. We use the size of your first input image. This node is to avoid having an input image size that is too large (such as 3000×3000 pixels), which could bring bad results." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 115, + "type": "VAEDecode", + "pos": [ + -259.239990234375, + 1497.972412109375 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 226 + }, + { + "name": "vae", + "type": "VAE", + "link": 243 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 234 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 117, + "type": "MaskToImage", + "pos": [ + 88.0047607421875, + 1383.150390625 + ], + "size": [ + 184.62362670898438, + 26 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 238 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 231 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "MaskToImage" + }, + "widgets_values": [] + }, + { + "id": 120, + "type": "VAEEncode", + "pos": [ + -244.5130157470703, + 1405.4102783203125 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 235 + }, + { + "name": "vae", + "type": "VAE", + "link": 236 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 226 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "VAEEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 121, + "type": "ImageScale", + "pos": [ + 43.72477340698242, + 1461.8167724609375 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 231 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 232 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 233 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 229 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "ImageScale" + }, + "widgets_values": [ + "nearest-exact", + 512, + 512, + "center" + ] + }, + { + "id": 122, + "type": "GetImageSize", + "pos": [ + -247.08106994628906, + 1597.323486328125 + ], + "size": [ + 140, + 124 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 234 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 227, + 232 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 228, + 233 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "GetImageSize" + }, + "widgets_values": [] + }, + { + "id": 93, + "type": "ImageScaleToTotalPixels", + "pos": [ + -230, + 890 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 177 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 224, + 225, + 235 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "ImageScaleToTotalPixels", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "upscale_method": true, + "megapixels": true + } + } + }, + "widgets_values": [ + "lanczos", + 1 + ] + }, + { + "id": 88, + "type": "VAEEncode", + "pos": [ + 264.1481018066406, + 899.3114624023438 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 239 + }, + { + "name": "vae", + "type": "VAE", + "link": 168 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 240 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "VAEEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 124, + "type": "SetLatentNoiseMask", + "pos": [ + 476.48956298828125, + 897.2662963867188 + ], + "size": [ + 180.74765014648438, + 46 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 240 + }, + { + "name": "mask", + "type": "MASK", + "link": 242 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 246 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 111, + "type": "TextEncodeQwenImageEditPlus", + "pos": [ + 220, + -90 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 205 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 207 + }, + { + "name": "image1", + "shape": 7, + "type": "IMAGE", + "link": 224 + }, + { + "name": "image2", + "shape": 7, + "type": "IMAGE", + "link": 219 + }, + { + "name": "image3", + "shape": 7, + "type": "IMAGE", + "link": 217 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 244 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "TextEncodeQwenImageEditPlus" + }, + "widgets_values": [ + "change the girl's cloth to red evening gown " + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 78, + "type": "LoadImage", + "pos": [ + -238.3336944580078, + 517.50244140625 + ], + "size": [ + 274.080078125, + 314.0000305175781 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 177, + 237 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 238 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoadImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "image": true, + "upload": true + } + } + }, + "widgets_values": [ + "Masked_Load_Me_in_Loader (9).png", + "image" + ] + }, + { + "id": 39, + "type": "VAELoader", + "pos": [ + -250, + 320 + ], + "size": [ + 330, + 60 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 76, + 168, + 206, + 207, + 236, + 243 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "qwen_image_vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/qwen_image_vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_vae.safetensors" + ] + }, + { + "id": 89, + "type": "LoraLoaderModelOnly", + "pos": [ + -240, + 40 + ], + "size": [ + 310, + 82 + ], + "flags": {}, + "order": 13, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 184 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 185 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoraLoaderModelOnly", + "models": [ + { + "name": "Qwen-Image-Lightning-4steps-V1.0.safetensors", + "url": "https://huggingface.co/lightx2v/Qwen-Image-Lightning/resolve/main/Qwen-Image-Lightning-4steps-V1.0.safetensors", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "lora_name": true, + "strength_model": true + } + } + }, + "widgets_values": [ + "Qwen-Image-Lightning-4steps-V1.0.safetensors", + 1 + ] + }, + { + "id": 75, + "type": "CFGNorm", + "pos": [ + 740, + -20 + ], + "size": [ + 290, + 60 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 141 + } + ], + "outputs": [ + { + "name": "patched_model", + "type": "MODEL", + "links": [ + 247 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "CFGNorm", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": { + "strength": true + } + } + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 37, + "type": "UNETLoader", + "pos": [ + -250, + -100 + ], + "size": [ + 330, + 90 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 184 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "qwen_image_edit_2509_fp8_e4m3fn.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/resolve/main/split_files/diffusion_models/qwen_image_edit_2509_fp8_e4m3fn.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_edit_2509_fp8_e4m3fn.safetensors", + "default" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1080, + -130 + ], + "size": [ + 210, + 46 + ], + "flags": { + "collapsed": false + }, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 248 + }, + { + "name": "vae", + "type": "VAE", + "link": 76 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 250 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "VAEDecode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 116, + "type": "ImageScale", + "pos": [ + 45.519248962402344, + 1207.10595703125 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 237 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 227 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 228 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 239, + 251 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "ImageScale" + }, + "widgets_values": [ + "lanczos", + 512, + 512, + "center" + ] + }, + { + "id": 118, + "type": "ImageToMask", + "pos": [ + 45.0244255065918, + 1635.2989501953125 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 229 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 242, + 252 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.51", + "Node name for S&R": "ImageToMask" + }, + "widgets_values": [ + "red" + ] + }, + { + "id": 126, + "type": "LanPaint_MaskBlend", + "pos": [ + 1087.578857421875, + -12.272896766662598 + ], + "size": [ + 210, + 98 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 251 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 250 + }, + { + "name": "mask", + "type": "MASK", + "link": 252 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 249 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712", + "Node name for S&R": "LanPaint_MaskBlend" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 125, + "type": "LanPaint_KSampler", + "pos": [ + 739.589599609375, + 84.47759246826172 + ], + "size": [ + 292.9582214355469, + 572 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 247 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 244 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 245 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 246 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 248 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "453b6c090e400123bb82f5bc0c97854031b09274", + "Node name for S&R": "LanPaint_KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 20, + 2.5, + "euler", + "simple", + 1, + 1, + "Image First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + }, + { + "id": 127, + "type": "SaveImage", + "pos": [ + 1090.8616943359375, + 133.32968139648438 + ], + "size": [ + 711.9912109375, + 634.4884643554688 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 249 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + } + ], + "links": [ + [ + 76, + 39, + 0, + 8, + 1, + "VAE" + ], + [ + 141, + 66, + 0, + 75, + 0, + "MODEL" + ], + [ + 168, + 39, + 0, + 88, + 1, + "VAE" + ], + [ + 177, + 78, + 0, + 93, + 0, + "IMAGE" + ], + [ + 184, + 37, + 0, + 89, + 0, + "MODEL" + ], + [ + 185, + 89, + 0, + 66, + 0, + "MODEL" + ], + [ + 204, + 38, + 0, + 110, + 0, + "CLIP" + ], + [ + 205, + 38, + 0, + 111, + 0, + "CLIP" + ], + [ + 206, + 39, + 0, + 110, + 1, + "VAE" + ], + [ + 207, + 39, + 0, + 111, + 1, + "VAE" + ], + [ + 217, + 108, + 0, + 111, + 4, + "IMAGE" + ], + [ + 218, + 108, + 0, + 110, + 4, + "IMAGE" + ], + [ + 219, + 106, + 0, + 111, + 3, + "IMAGE" + ], + [ + 220, + 106, + 0, + 110, + 3, + "IMAGE" + ], + [ + 224, + 93, + 0, + 111, + 2, + "IMAGE" + ], + [ + 225, + 93, + 0, + 110, + 2, + "IMAGE" + ], + [ + 226, + 120, + 0, + 115, + 0, + "LATENT" + ], + [ + 227, + 122, + 0, + 116, + 1, + "INT" + ], + [ + 228, + 122, + 1, + 116, + 2, + "INT" + ], + [ + 229, + 121, + 0, + 118, + 0, + "IMAGE" + ], + [ + 231, + 117, + 0, + 121, + 0, + "IMAGE" + ], + [ + 232, + 122, + 0, + 121, + 1, + "INT" + ], + [ + 233, + 122, + 1, + 121, + 2, + "INT" + ], + [ + 234, + 115, + 0, + 122, + 0, + "IMAGE" + ], + [ + 235, + 93, + 0, + 120, + 0, + "IMAGE" + ], + [ + 236, + 39, + 0, + 120, + 1, + "VAE" + ], + [ + 237, + 78, + 0, + 116, + 0, + "IMAGE" + ], + [ + 238, + 78, + 1, + 117, + 0, + "MASK" + ], + [ + 239, + 116, + 0, + 88, + 0, + "IMAGE" + ], + [ + 240, + 88, + 0, + 124, + 0, + "LATENT" + ], + [ + 242, + 118, + 0, + 124, + 1, + "MASK" + ], + [ + 243, + 39, + 0, + 115, + 1, + "VAE" + ], + [ + 244, + 111, + 0, + 125, + 1, + "CONDITIONING" + ], + [ + 245, + 110, + 0, + 125, + 2, + "CONDITIONING" + ], + [ + 246, + 124, + 0, + 125, + 3, + "LATENT" + ], + [ + 247, + 75, + 0, + 125, + 0, + "MODEL" + ], + [ + 248, + 125, + 0, + 8, + 0, + "LATENT" + ], + [ + 249, + 126, + 0, + 127, + 0, + "IMAGE" + ], + [ + 250, + 8, + 0, + 126, + 1, + "IMAGE" + ], + [ + 251, + 116, + 0, + 126, + 0, + "IMAGE" + ], + [ + 252, + 118, + 0, + 126, + 2, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "Step1 - Load models", + "bounding": [ + -270, + -170, + 370, + 570 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Step 2 - Upload image for editing", + "bounding": [ + -270, + 430, + 970, + 550 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Step 4 - Prompt", + "bounding": [ + 130, + -170, + 570, + 570 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Step3 - Image Size", + "bounding": [ + 730, + 780, + 310, + 200 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6413348663336585, + "offset": [ + -528.1379465475136, + 63.45086652498852 + ] + }, + "frontendVersion": "1.26.15", + "ue_links": [], + "links_added_by_ue": [], + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Qwen_Image_Inpaint.jpg b/LanPaint/example_workflows/Qwen_Image_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..64aca9ee05d5e22f7de091cea6f9b24c1a573b9e --- /dev/null +++ b/LanPaint/example_workflows/Qwen_Image_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ad37bbc61db9627a5e1c8a98a35675cca813ebf699f2ebe051414fe70ea2c7 +size 942926 diff --git a/LanPaint/example_workflows/Qwen_Image_Inpaint.json b/LanPaint/example_workflows/Qwen_Image_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..99af8dadb107e761ea480b8e967f518a014cd773 --- /dev/null +++ b/LanPaint/example_workflows/Qwen_Image_Inpaint.json @@ -0,0 +1,806 @@ +{ + "id": "11cce4ab-536b-4f42-a95c-0be437d04ace", + "revision": 0, + "last_node_id": 128, + "last_link_id": 338, + "nodes": [ + { + "id": 74, + "type": "LanPaint_KSampler", + "pos": [ + 276.219970703125, + 179.55892944335938 + ], + "size": [ + 388.97625732421875, + 572 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 325 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 323 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 324 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 332 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 334 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "56bd6c04e89124cd06682b304245d6ddf8b20522", + "Node name for S&R": "LanPaint_KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 20, + 4, + "euler", + "simple", + 1, + 5, + "Image First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + }, + { + "id": 113, + "type": "SaveImage", + "pos": [ + 807.1268310546875, + 868.395263671875 + ], + "size": [ + 311.2532653808594, + 484.7096252441406 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 338 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 117, + "type": "CLIPLoader", + "pos": [ + -824.4296875, + 177.9814910888672 + ], + "size": [ + 330, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 318, + 319 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "qwen_image", + "default" + ] + }, + { + "id": 118, + "type": "VAELoader", + "pos": [ + -824.4296875, + 327.9817199707031 + ], + "size": [ + 330, + 60 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 329, + 335 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "qwen_image_vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/qwen_image_vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_vae.safetensors" + ] + }, + { + "id": 121, + "type": "CLIPTextEncode", + "pos": [ + -454.4298095703125, + 247.9816436767578 + ], + "size": [ + 425.27801513671875, + 180.6060791015625 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 319 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 324 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPTextEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + " low quality, bad anatomy, extra digits, missing digits, extra limbs, missing limbs" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 122, + "type": "ModelSamplingAuraFlow", + "pos": [ + -34.14249038696289, + -43.64523696899414 + ], + "size": [ + 300, + 58 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 320 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 325 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "ModelSamplingAuraFlow", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 3.5 + ] + }, + { + "id": 119, + "type": "UNETLoader", + "pos": [ + -824.4296875, + 37.98154830932617 + ], + "size": [ + 330, + 90 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 320 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "qwen_image_fp8_e4m3fn.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/diffusion_models/qwen_image_fp8_e4m3fn.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_fp8_e4m3fn.safetensors", + "default" + ] + }, + { + "id": 124, + "type": "VAEEncode", + "pos": [ + -530.8583984375, + 708.7066650390625 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 326 + }, + { + "name": "vae", + "type": "VAE", + "link": 329 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 327 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [] + }, + { + "id": 125, + "type": "SetLatentNoiseMask", + "pos": [ + -234.4196014404297, + 705.1629638671875 + ], + "size": [ + 264.5999755859375, + 46 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 327 + }, + { + "name": "mask", + "type": "MASK", + "link": 328 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 332 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 127, + "type": "LanPaint_MaskBlend", + "pos": [ + 405.3840637207031, + 938.8120727539062 + ], + "size": [ + 210, + 98 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 336 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 333 + }, + { + "name": "mask", + "type": "MASK", + "link": 337 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 338 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712", + "Node name for S&R": "LanPaint_MaskBlend" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 126, + "type": "VAEDecode", + "pos": [ + 115.07575225830078, + 878.4630737304688 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 334 + }, + { + "name": "vae", + "type": "VAE", + "link": 335 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 333 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 123, + "type": "LoadImage", + "pos": [ + -543.5358276367188, + 851.759765625 + ], + "size": [ + 262.12347412109375, + 487.22296142578125 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 326, + 336 + ] + }, + { + "name": "MASK", + "type": "MASK", + "slot_index": 1, + "links": [ + 328, + 337 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "Masked_Load_Me_in_Loader (7).png", + "image" + ] + }, + { + "id": 120, + "type": "CLIPTextEncode", + "pos": [ + -454.86480712890625, + 41.89194869995117 + ], + "size": [ + 422.84503173828125, + 164.31304931640625 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 318 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 323 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPTextEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Cyberpunk-style Einstein portrait: He wears a sleek black coat with glowing cyan circuit patterns, silver-rimmed cybernetic glasses (lenses display faint data streams), and his hair has subtle neon blue highlights. His expression is calm, with a faint smile. Behind him: a dark, rain-washed cybercity backdrop—towering skyscrapers with flickering holographic ads, wet pavement reflecting neon pink/magenta lights. In front of him: giant, glowing white 3D text of \"LanPaint\", with electric blue energy pulses swirling around the equation. Cinematic lighting, hyper-detailed textures, rain droplets visible in the air." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 128, + "type": "MarkdownNote", + "pos": [ + 715.929931640625, + 371.1071472167969 + ], + "size": [ + 300, + 190 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "KSampler settings", + "properties": {}, + "widgets_values": [ + "Decrease **LanPaint_NumSteps** for faster generation. \n" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 318, + 117, + 0, + 120, + 0, + "CLIP" + ], + [ + 319, + 117, + 0, + 121, + 0, + "CLIP" + ], + [ + 320, + 119, + 0, + 122, + 0, + "MODEL" + ], + [ + 323, + 120, + 0, + 74, + 1, + "CONDITIONING" + ], + [ + 324, + 121, + 0, + 74, + 2, + "CONDITIONING" + ], + [ + 325, + 122, + 0, + 74, + 0, + "MODEL" + ], + [ + 326, + 123, + 0, + 124, + 0, + "IMAGE" + ], + [ + 327, + 124, + 0, + 125, + 0, + "LATENT" + ], + [ + 328, + 123, + 1, + 125, + 1, + "MASK" + ], + [ + 329, + 118, + 0, + 124, + 1, + "VAE" + ], + [ + 332, + 125, + 0, + 74, + 3, + "LATENT" + ], + [ + 333, + 126, + 0, + 127, + 1, + "IMAGE" + ], + [ + 334, + 74, + 0, + 126, + 0, + "LATENT" + ], + [ + 335, + 118, + 0, + 126, + 1, + "VAE" + ], + [ + 336, + 123, + 0, + 127, + 0, + "IMAGE" + ], + [ + 337, + 123, + 1, + 127, + 2, + "MASK" + ], + [ + 338, + 127, + 0, + 113, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.7162766973052638, + "offset": [ + 1084.0595529886727, + 5.084234529384386 + ] + }, + "frontendVersion": "1.25.10", + "node_versions": { + "comfy-core": "0.3.18", + "LanPaint": "0f509469ed2cd60c6032f739e282aad5dfc06166" + }, + "groupNodes": {} + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Qwen_Image_Outpaint.jpg b/LanPaint/example_workflows/Qwen_Image_Outpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4b94c00cd2ba9019ec9a5f0fce83c6171be3e530 --- /dev/null +++ b/LanPaint/example_workflows/Qwen_Image_Outpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:464f2cd7bfcef06bfc32f79b8d08f8c064048b5cc2e83b3eb58da4eb5139143d +size 829645 diff --git a/LanPaint/example_workflows/Qwen_Image_Outpaint.json b/LanPaint/example_workflows/Qwen_Image_Outpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..dc2e1c0f7cec6a83a7627ba14b1a94dc45b67378 --- /dev/null +++ b/LanPaint/example_workflows/Qwen_Image_Outpaint.json @@ -0,0 +1,909 @@ +{ + "id": "11cce4ab-536b-4f42-a95c-0be437d04ace", + "revision": 0, + "last_node_id": 136, + "last_link_id": 351, + "nodes": [ + { + "id": 74, + "type": "LanPaint_KSampler", + "pos": [ + 276.219970703125, + 179.55892944335938 + ], + "size": [ + 388.97625732421875, + 572 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 336 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 334 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 335 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 345 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 347 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "56bd6c04e89124cd06682b304245d6ddf8b20522", + "Node name for S&R": "LanPaint_KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 20, + 4, + "euler", + "simple", + 1, + 5, + "Image First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + }, + { + "id": 113, + "type": "SaveImage", + "pos": [ + 807.1268310546875, + 868.395263671875 + ], + "size": [ + 311.2532653808594, + 484.7096252441406 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 351 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 123, + "type": "CLIPLoader", + "pos": [ + -730.1405029296875, + 183.70140075683594 + ], + "size": [ + 330, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 329, + 330 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "qwen_image", + "default" + ] + }, + { + "id": 124, + "type": "VAELoader", + "pos": [ + -730.1405029296875, + 333.70147705078125 + ], + "size": [ + 330, + 60 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 342, + 348 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "qwen_image_vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/qwen_image_vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_vae.safetensors" + ] + }, + { + "id": 125, + "type": "UNETLoader", + "pos": [ + -730.1405029296875, + 43.7014274597168 + ], + "size": [ + 330, + 90 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 331 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "qwen_image_fp8_e4m3fn.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/diffusion_models/qwen_image_fp8_e4m3fn.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "qwen_image_fp8_e4m3fn.safetensors", + "default" + ] + }, + { + "id": 127, + "type": "CLIPTextEncode", + "pos": [ + -360.1405029296875, + 253.70147705078125 + ], + "size": [ + 425.27801513671875, + 180.6060791015625 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 330 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 335 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPTextEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + " low quality, bad anatomy, extra digits, missing digits, extra limbs, missing limbs" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 128, + "type": "ModelSamplingAuraFlow", + "pos": [ + 60.14683151245117, + -37.92536544799805 + ], + "size": [ + 300, + 58 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 331 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 336 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "ModelSamplingAuraFlow", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 3.5 + ] + }, + { + "id": 126, + "type": "CLIPTextEncode", + "pos": [ + -360.57550048828125, + 47.6118278503418 + ], + "size": [ + 422.84503173828125, + 164.31304931640625 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 329 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 334 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPTextEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Cyberpunk-inspired portrait of a beautiful young woman with ethereal features: She has long, flowing silver hair with glowing purple neon streaks, wearing a form-fitting black leather jacket adorned with holographic circuit designs in electric blue, and augmented reality earrings that project faint digital fractals. Her eyes are piercing emerald green with subtle cybernetic enhancements showing data overlays, and her expression is mysterious yet alluring, with a subtle smirk. Behind her: a foggy, neon-lit futuristic alleyway—crumbling brick walls covered in vibrant graffiti and flickering LED signs, puddles on the ground reflecting turquoise and violet lights from overhead drones. Dramatic volumetric lighting, ultra-realistic details, mist particles in the air." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 130, + "type": "ImagePadForOutpaint", + "pos": [ + -437.0110168457031, + 895.0103759765625 + ], + "size": [ + 210, + 174 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 337 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 338, + 349 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 339 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.39", + "Node name for S&R": "ImagePadForOutpaint", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 200, + 200, + 200, + 200, + 20 + ] + }, + { + "id": 131, + "type": "VAEEncode", + "pos": [ + -455.7289123535156, + 723.4658203125 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 338 + }, + { + "name": "vae", + "type": "VAE", + "link": 342 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 340 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [] + }, + { + "id": 132, + "type": "ThresholdMask", + "pos": [ + -67.46508026123047, + 903.83642578125 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 339 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 341, + 350 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.38", + "Node name for S&R": "ThresholdMask" + }, + "widgets_values": [ + 0.010000000000000002 + ] + }, + { + "id": 133, + "type": "SetLatentNoiseMask", + "pos": [ + -152.71241760253906, + 712.6437377929688 + ], + "size": [ + 264.5999755859375, + 46 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 340 + }, + { + "name": "mask", + "type": "MASK", + "link": 341 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 345 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 129, + "type": "LoadImage", + "pos": [ + -789.0392456054688, + 827.515380859375 + ], + "size": [ + 295, + 399 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 337 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "LoadImage", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "ComfyUI_07699_.png", + "image" + ] + }, + { + "id": 134, + "type": "VAEDecode", + "pos": [ + 256.8516540527344, + 900.2234497070312 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 347 + }, + { + "name": "vae", + "type": "VAE", + "link": 348 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 346 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 135, + "type": "LanPaint_MaskBlend", + "pos": [ + 547.1599731445312, + 960.5724487304688 + ], + "size": [ + 210, + 98 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 349 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 346 + }, + { + "name": "mask", + "type": "MASK", + "link": 350 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 351 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712", + "Node name for S&R": "LanPaint_MaskBlend" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 136, + "type": "MarkdownNote", + "pos": [ + 728.9194946289062, + 332.5254821777344 + ], + "size": [ + 300, + 190 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "KSampler settings", + "properties": {}, + "widgets_values": [ + "Decrease **LanPaint_NumSteps** for faster generation. \n" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 329, + 123, + 0, + 126, + 0, + "CLIP" + ], + [ + 330, + 123, + 0, + 127, + 0, + "CLIP" + ], + [ + 331, + 125, + 0, + 128, + 0, + "MODEL" + ], + [ + 334, + 126, + 0, + 74, + 1, + "CONDITIONING" + ], + [ + 335, + 127, + 0, + 74, + 2, + "CONDITIONING" + ], + [ + 336, + 128, + 0, + 74, + 0, + "MODEL" + ], + [ + 337, + 129, + 0, + 130, + 0, + "IMAGE" + ], + [ + 338, + 130, + 0, + 131, + 0, + "IMAGE" + ], + [ + 339, + 130, + 1, + 132, + 0, + "MASK" + ], + [ + 340, + 131, + 0, + 133, + 0, + "LATENT" + ], + [ + 341, + 132, + 0, + 133, + 1, + "MASK" + ], + [ + 342, + 124, + 0, + 131, + 1, + "VAE" + ], + [ + 345, + 133, + 0, + 74, + 3, + "LATENT" + ], + [ + 346, + 134, + 0, + 135, + 1, + "IMAGE" + ], + [ + 347, + 74, + 0, + 134, + 0, + "LATENT" + ], + [ + 348, + 124, + 0, + 134, + 1, + "VAE" + ], + [ + 349, + 130, + 0, + 135, + 0, + "IMAGE" + ], + [ + 350, + 132, + 0, + 135, + 2, + "MASK" + ], + [ + 351, + 135, + 0, + 113, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6087272163919795, + "offset": [ + 1617.0504979084035, + 167.0583828525509 + ] + }, + "frontendVersion": "1.25.10", + "node_versions": { + "comfy-core": "0.3.18", + "LanPaint": "0f509469ed2cd60c6032f739e282aad5dfc06166" + }, + "groupNodes": {} + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/SDXL_Inpaint.jpg b/LanPaint/example_workflows/SDXL_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a8ef6b38e7626f04f4d79f72cfcf404193d754b0 --- /dev/null +++ b/LanPaint/example_workflows/SDXL_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32f61d5899e8b6e741f992fe163e5d18cd28f72d4fe49f3b348a465ffb1268bc +size 1406666 diff --git a/LanPaint/example_workflows/SDXL_Inpaint.json b/LanPaint/example_workflows/SDXL_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..8746f0f9b9716f1d9484036e379b49e45003e1f6 --- /dev/null +++ b/LanPaint/example_workflows/SDXL_Inpaint.json @@ -0,0 +1,735 @@ +{ + "id": "26fb90cb-eb4a-422e-97d0-8b84dd6c3302", + "revision": 0, + "last_node_id": 75, + "last_link_id": 192, + "nodes": [ + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 333.06903076171875, + 249.68698120117188 + ], + "size": [ + 422.84503173828125, + 164.31304931640625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 81 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 184 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "1girl, blue shirt, masterpiece, high score, great score, absurdres" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 335.06903076171875, + 462.68701171875 + ], + "size": [ + 425.27801513671875, + 180.6060791015625 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 82 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 185 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "lowres, bad anatomy, bad hands, text, error, missing finger, extra digits, fewer digits, cropped, worst quality, low quality, low score, bad score, average score, signature, watermark, username, blurry, nude, NSFW" + ] + }, + { + "id": 29, + "type": "CheckpointLoaderSimple", + "pos": [ + -62.627220153808594, + 407.01416015625 + ], + "size": [ + 315, + 98 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 183 + ] + }, + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 1, + "links": [ + 81, + 82 + ] + }, + { + "name": "VAE", + "type": "VAE", + "slot_index": 2, + "links": [ + 84, + 157 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "animagineXL40_v4Opt.safetensors" + ] + }, + { + "id": 48, + "type": "SaveImage", + "pos": [ + 1091.036376953125, + 1158.526611328125 + ], + "size": [ + 311.2532653808594, + 484.7096252441406 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 103 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1211.46484375, + 1065.318359375 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 187 + }, + { + "name": "vae", + "type": "VAE", + "link": 84 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 103, + 189 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 20, + "type": "LoadImage", + "pos": [ + 45.70227813720703, + 1147.2928466796875 + ], + "size": [ + 262.12347412109375, + 487.22296142578125 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 156, + 190 + ] + }, + { + "name": "MASK", + "type": "MASK", + "slot_index": 1, + "links": [ + 155, + 191 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "clipspace/clipspace-mask-2620525.399999976.png [input]", + "image" + ] + }, + { + "id": 75, + "type": "SaveImage", + "pos": [ + 1845.7059326171875, + 1044.06201171875 + ], + "size": [ + 311.2532653808594, + 484.7096252441406 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 188 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 73, + "type": "LanPaint_KSampler", + "pos": [ + 996.4982299804688, + 299.599365234375 + ], + "size": [ + 413.6495666503906, + 572 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 183 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 184 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 185 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 186 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 187 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "56bd6c04e89124cd06682b304245d6ddf8b20522", + "Node name for S&R": "LanPaint_KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 5, + "euler", + "karras", + 1, + 5, + "Image First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + }, + { + "id": 74, + "type": "LanPaint_MaskBlend", + "pos": [ + 1512.2730712890625, + 1176.3270263671875 + ], + "size": [ + 210, + 98 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 190 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 189 + }, + { + "name": "mask", + "type": "MASK", + "link": 191 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 188 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712", + "Node name for S&R": "LanPaint_MaskBlend" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 66, + "type": "SetLatentNoiseMask", + "pos": [ + 480.30780029296875, + 821.3880004882812 + ], + "size": [ + 264.5999755859375, + 46 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 192 + }, + { + "name": "mask", + "type": "MASK", + "link": 155 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 186 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 65, + "type": "VAEEncode", + "pos": [ + 206.43878173828125, + 818.9985961914062 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 156 + }, + { + "name": "vae", + "type": "VAE", + "link": 157 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 192 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 81, + 29, + 1, + 6, + 0, + "CLIP" + ], + [ + 82, + 29, + 1, + 7, + 0, + "CLIP" + ], + [ + 84, + 29, + 2, + 8, + 1, + "VAE" + ], + [ + 103, + 8, + 0, + 48, + 0, + "IMAGE" + ], + [ + 155, + 20, + 1, + 66, + 1, + "MASK" + ], + [ + 156, + 20, + 0, + 65, + 0, + "IMAGE" + ], + [ + 157, + 29, + 2, + 65, + 1, + "VAE" + ], + [ + 183, + 29, + 0, + 73, + 0, + "MODEL" + ], + [ + 184, + 6, + 0, + 73, + 1, + "CONDITIONING" + ], + [ + 185, + 7, + 0, + 73, + 2, + "CONDITIONING" + ], + [ + 186, + 66, + 0, + 73, + 3, + "LATENT" + ], + [ + 187, + 73, + 0, + 8, + 0, + "LATENT" + ], + [ + 188, + 74, + 0, + 75, + 0, + "IMAGE" + ], + [ + 189, + 8, + 0, + 74, + 1, + "IMAGE" + ], + [ + 190, + 20, + 0, + 74, + 0, + "IMAGE" + ], + [ + 191, + 20, + 1, + 74, + 2, + "MASK" + ], + [ + 192, + 65, + 0, + 66, + 0, + "LATENT" + ] + ], + "groups": [ + { + "id": 1, + "title": "Mask image for inpainting.", + "bounding": [ + 36.04227828979492, + 989.7313232421875, + 278.89093017578125, + 669.3414916992188 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Convert Latents for LanPaint", + "bounding": [ + 286.0640563964844, + 714.343505859375, + 489.16796875, + 197.81044006347656 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model and Set Prompts", + "bounding": [ + -78.9311294555664, + 176.08712768554688, + 1006.1721801757812, + 514.258544921875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Inpaint with the LanPaint KSampler", + "bounding": [ + 960.8922729492188, + 179.17588806152344, + 474.8909606933594, + 630.4742431640625 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "LanPaint OutPut", + "bounding": [ + 1085.6029052734375, + 994.0775756835938, + 345.4561767578125, + 669.4969482421875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 11, + "title": "LanPaint", + "bounding": [ + -262.59381103515625, + 140.46656799316406, + 1737.328857421875, + 797.4443359375 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 13, + "title": "Paste back the original to preserve it exactly, if you want", + "bounding": [ + 1491.586181640625, + 984.3547973632812, + 749.66455078125, + 637.360595703125 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6209213230591556, + "offset": [ + 850.0803535605011, + 86.6432096141053 + ] + }, + "frontendVersion": "1.25.10", + "node_versions": { + "comfy-core": "0.3.18", + "LanPaint": "0f509469ed2cd60c6032f739e282aad5dfc06166" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Z_image_Inpaint.jpg b/LanPaint/example_workflows/Z_image_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e2dde30a6d822bd107c8af25969370980f07bab3 --- /dev/null +++ b/LanPaint/example_workflows/Z_image_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf29021eee787d5b6e81f094719206ed06db75335e992db94454b68b8330121 +size 1151241 diff --git a/LanPaint/example_workflows/Z_image_Inpaint.json b/LanPaint/example_workflows/Z_image_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..7f614d6ce02a0f0e41afd49b2954f664227e5c93 --- /dev/null +++ b/LanPaint/example_workflows/Z_image_Inpaint.json @@ -0,0 +1,1475 @@ +{ + "id": "9ae6082b-c7f4-433c-9971-7a8f65a3ea65", + "revision": 0, + "last_node_id": 73, + "last_link_id": 95, + "nodes": [ + { + "id": 39, + "type": "CLIPLoader", + "pos": [ + 158.31605577680375, + 488.30506119064967 + ], + "size": [ + 323.734375, + 179.390625 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 44 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "models": [ + { + "name": "qwen_3_4b.safetensors", + "url": "https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/text_encoders/qwen_3_4b.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "lumina2", + "default" + ] + }, + { + "id": 40, + "type": "VAELoader", + "pos": [ + 159.30157212272297, + 693.9279006020886 + ], + "size": [ + 323.734375, + 111.390625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 39, + 70, + 86, + 87 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "models": [ + { + "name": "ae.safetensors", + "url": "https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/vae/ae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "ae.safetensors" + ] + }, + { + "id": 42, + "type": "ConditioningZeroOut", + "pos": [ + 794.3164307166835, + 836.3052371197768 + ], + "size": [ + 236.984375, + 77.421875 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 36 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 64 + ] + } + ], + "properties": { + "Node name for S&R": "ConditioningZeroOut", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [] + }, + { + "id": 41, + "type": "EmptySD3LatentImage", + "pos": [ + 158.31605577680375, + 848.3047505883683 + ], + "size": [ + 311.734375, + 179.390625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "Node name for S&R": "EmptySD3LatentImage", + "cnr_id": "comfy-core", + "ver": "0.3.64", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 47, + "type": "ModelSamplingAuraFlow", + "pos": [ + 1106.844558468847, + 300.1767719883135 + ], + "size": [ + 371.875, + 111.390625 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 60 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 62 + ] + } + ], + "properties": { + "Node name for S&R": "ModelSamplingAuraFlow", + "cnr_id": "comfy-core", + "ver": "0.3.64", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 43, + "type": "VAEDecode", + "pos": [ + 1490.0007839701339, + 169.99989031781615 + ], + "size": [ + 251.734375, + 101.421875 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 66 + }, + { + "name": "vae", + "type": "VAE", + "link": 39 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 73, + 93 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.3.64", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [] + }, + { + "id": 45, + "type": "CLIPTextEncode", + "pos": [ + 542.3159768994137, + 332.3053621604172 + ], + "size": [ + 491.734375, + 479.734375 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 44 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 36, + 63 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "Latina female with thick wavy hair, white shirt, harbor boats and pastel houses behind. Breezy seaside light, warm tones, cinematic close-up." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 48, + "type": "LoraLoaderModelOnly", + "pos": [ + 553.9994917632011, + 133.99988695553816 + ], + "size": [ + 443.75, + 145.390625 + ], + "flags": {}, + "order": 10, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 54 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 60 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.3.75", + "models": [ + { + "name": "pixel_art_style_z_image_turbo.safetensors", + "url": "https://huggingface.co/tarn59/pixel_art_style_lora_z_image_turbo/resolve/main/pixel_art_style_z_image_turbo.safetensors", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "pixel_art_style_z_image_turbo.safetensors", + 1 + ] + }, + { + "id": 35, + "type": "MarkdownNote", + "pos": [ + -466.00011584916956, + 289.9999302108306 + ], + "size": [ + 587.734375, + 770.859375 + ], + "flags": { + "collapsed": false + }, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model link", + "properties": {}, + "widgets_values": [ + "## Report workflow issue\n\nIf you found any issues when running this workflow, [report template issue here](https://github.com/Comfy-Org/workflow_templates/issues)\n\n\n## Model links\n\n**text_encoders**\n\n- [qwen_3_4b.safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/text_encoders/qwen_3_4b.safetensors)\n\n**loras**\n\n- [pixel_art_style_z_image_turbo.safetensors](https://huggingface.co/tarn59/pixel_art_style_lora_z_image_turbo/resolve/main/pixel_art_style_z_image_turbo.safetensors)\n\n**diffusion_models**\n\n- [z_image_turbo_bf16.safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/diffusion_models/z_image_turbo_bf16.safetensors)\n\n**vae**\n\n- [ae.safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/vae/ae.safetensors)\n\n\nModel Storage Location\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ └── qwen_3_4b.safetensors\n│ ├── 📂 loras/\n│ │ └── pixel_art_style_z_image_turbo.safetensors\n│ ├── 📂 diffusion_models/\n│ │ └── z_image_turbo_bf16.safetensors\n│ └── 📂 vae/\n│ └── ae.safetensors\n```\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 56, + "type": "LanPaint_KSampler", + "pos": [ + 1280.6434468860161, + 480.42471689895115 + ], + "size": [ + 479.765625, + 887.59375 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 62 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 63 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 64 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 71 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 66 + ] + } + ], + "properties": { + "Node name for S&R": "LanPaint_KSampler", + "cnr_id": "LanPaint", + "ver": "6109df6591a4cf2bc9d3b113d03f7297fa9248e9" + }, + "widgets_values": [ + 880311146947153, + "randomize", + 9, + 1, + "euler", + "simple", + 1, + 5, + "Image First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", + "🖼️ Image Inpainting" + ] + }, + { + "id": 57, + "type": "LoadImage", + "pos": [ + 9.292593918969033, + 1110.6054508670231 + ], + "size": [ + 328.65625, + 686.171875 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 85, + 88 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 89 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "cnr_id": "comfy-core", + "ver": "0.3.59", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + }, + "image": "clipspace/clipspace-painted-masked-1764827669471.png [input]" + }, + "widgets_values": [ + "clipspace/clipspace-painted-masked-1764827669471.png [input]", + "image" + ] + }, + { + "id": 58, + "type": "VAEEncode", + "pos": [ + 1701.3714934284335, + 1667.0261553802961 + ], + "size": [ + 225, + 101.421875 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 90 + }, + { + "name": "vae", + "type": "VAE", + "link": 70 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "VAEEncode", + "cnr_id": "comfy-core", + "ver": "0.3.59", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 59, + "type": "SetLatentNoiseMask", + "pos": [ + 2169.0494285384316, + 1637.2004308575815 + ], + "size": [ + 225, + 101.421875 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 67 + }, + { + "name": "mask", + "type": "MASK", + "link": 91 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "SetLatentNoiseMask", + "cnr_id": "comfy-core", + "ver": "0.3.59", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 60, + "type": "LanPaint_MaskBlend", + "pos": [ + 2744.756891034133, + 1245.7532005683747 + ], + "size": [ + 251.84375, + 159.40625 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 94 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 73 + }, + { + "name": "mask", + "type": "MASK", + "link": 95 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 72 + ] + } + ], + "properties": { + "Node name for S&R": "LanPaint_MaskBlend", + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 61, + "type": "SaveImage", + "pos": [ + 3313.9622863771715, + 1191.2159032348995 + ], + "size": [ + 1304.078125, + 1434.015625 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 72 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 62, + "type": "VAEDecode", + "pos": [ + 909.4390596825519, + 2032.1553145051257 + ], + "size": [ + 251.90625, + 101.421875 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 76 + }, + { + "name": "vae", + "type": "VAE", + "link": 87 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [] + }, + { + "id": 64, + "type": "MaskToImage", + "pos": [ + 1280.1054231177059, + 1881.6714778315163 + ], + "size": [ + 225, + 77.421875 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 89 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage", + "cnr_id": "comfy-core", + "ver": "0.3.51" + }, + "widgets_values": [] + }, + { + "id": 67, + "type": "VAEEncode", + "pos": [ + 927.11123002246, + 1921.0804280162624 + ], + "size": [ + 225, + 101.421875 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 85 + }, + { + "name": "vae", + "type": "VAE", + "link": 86 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 76 + ] + } + ], + "properties": { + "Node name for S&R": "VAEEncode", + "cnr_id": "comfy-core", + "ver": "0.3.50", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 68, + "type": "ImageScale", + "pos": [ + 1272.9968314098537, + 1988.7684995304935 + ], + "size": [ + 323.90625, + 213.375 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 81 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 82 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 83 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "ImageScale", + "cnr_id": "comfy-core", + "ver": "0.3.52" + }, + "widgets_values": [ + "nearest-exact", + 512, + 512, + "center" + ] + }, + { + "id": 69, + "type": "GetImageSize", + "pos": [ + 924.0300715130902, + 2151.376631256835 + ], + "size": [ + 225, + 190.171875 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 84 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 77, + 82 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 78, + 83 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.3.52" + }, + "widgets_values": [ + "width: 1024, height: 1024\n batch size: 1" + ] + }, + { + "id": 73, + "type": "PreviewImage", + "pos": [ + 2226.506787053337, + 107.57278250350078 + ], + "size": [ + 225, + 246 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 93 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.3.76" + }, + "widgets_values": [] + }, + { + "id": 63, + "type": "ImageScale", + "pos": [ + 1283.837511214967, + 1657.4367336671903 + ], + "size": [ + 323.90625, + 213.375 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 88 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 77 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 78 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 90, + 94 + ] + } + ], + "properties": { + "Node name for S&R": "ImageScale", + "cnr_id": "comfy-core", + "ver": "0.3.52" + }, + "widgets_values": [ + "area", + 512, + 512, + "center" + ] + }, + { + "id": 65, + "type": "ImageToMask", + "pos": [ + 1274.556170930292, + 2196.94755423526 + ], + "size": [ + 323.90625, + 111.390625 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 79 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 91, + 95 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToMask", + "cnr_id": "comfy-core", + "ver": "0.3.51" + }, + "widgets_values": [ + "red" + ] + }, + { + "id": 46, + "type": "UNETLoader", + "pos": [ + 159.05646022112083, + 288.62105108441466 + ], + "size": [ + 323.734375, + 145.390625 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 54 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "models": [ + { + "name": "z_image_turbo_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/diffusion_models/z_image_turbo_bf16.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "default" + ] + }, + { + "id": 72, + "type": "Note", + "pos": [ + 1288.7909707861209, + 1503.5796522859714 + ], + "size": [ + 225, + 109.765625 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Reshape image to suit VAE " + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 36, + 45, + 0, + 42, + 0, + "CONDITIONING" + ], + [ + 39, + 40, + 0, + 43, + 1, + "VAE" + ], + [ + 44, + 39, + 0, + 45, + 0, + "CLIP" + ], + [ + 54, + 46, + 0, + 48, + 0, + "MODEL" + ], + [ + 60, + 48, + 0, + 47, + 0, + "MODEL" + ], + [ + 62, + 47, + 0, + 56, + 0, + "MODEL" + ], + [ + 63, + 45, + 0, + 56, + 1, + "CONDITIONING" + ], + [ + 64, + 42, + 0, + 56, + 2, + "CONDITIONING" + ], + [ + 66, + 56, + 0, + 43, + 0, + "LATENT" + ], + [ + 67, + 58, + 0, + 59, + 0, + "LATENT" + ], + [ + 70, + 40, + 0, + 58, + 1, + "VAE" + ], + [ + 71, + 59, + 0, + 56, + 3, + "LATENT" + ], + [ + 72, + 60, + 0, + 61, + 0, + "IMAGE" + ], + [ + 73, + 43, + 0, + 60, + 1, + "IMAGE" + ], + [ + 76, + 67, + 0, + 62, + 0, + "LATENT" + ], + [ + 77, + 69, + 0, + 63, + 1, + "INT" + ], + [ + 78, + 69, + 1, + 63, + 2, + "INT" + ], + [ + 79, + 68, + 0, + 65, + 0, + "IMAGE" + ], + [ + 81, + 64, + 0, + 68, + 0, + "IMAGE" + ], + [ + 82, + 69, + 0, + 68, + 1, + "INT" + ], + [ + 83, + 69, + 1, + 68, + 2, + "INT" + ], + [ + 84, + 62, + 0, + 69, + 0, + "IMAGE" + ], + [ + 85, + 57, + 0, + 67, + 0, + "IMAGE" + ], + [ + 86, + 40, + 0, + 67, + 1, + "VAE" + ], + [ + 87, + 40, + 0, + 62, + 1, + "VAE" + ], + [ + 88, + 57, + 0, + 63, + 0, + "IMAGE" + ], + [ + 89, + 57, + 1, + 64, + 0, + "MASK" + ], + [ + 90, + 63, + 0, + 58, + 0, + "IMAGE" + ], + [ + 91, + 65, + 0, + 59, + 1, + "MASK" + ], + [ + 93, + 43, + 0, + 73, + 0, + "IMAGE" + ], + [ + 94, + 63, + 0, + 60, + 0, + "IMAGE" + ], + [ + 95, + 65, + 0, + 60, + 2, + "MASK" + ] + ], + "groups": [ + { + "id": 2, + "title": "Step2 - Image size", + "bounding": [ + 146, + 770, + 348, + 240 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Step3 - Prompt", + "bounding": [ + 518, + 254, + 540, + 648 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Step1 - Load models", + "bounding": [ + 146, + 254, + 348, + 496.32 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "Ctrl-B to enable LoRA input", + "bounding": [ + 518, + 50, + 528, + 192 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.3800835362432756, + "offset": [ + -902.897494105397, + -396.4805064253962 + ] + }, + "frontendVersion": "1.33.14", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "workflowRendererVersion": "Vue" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/Z_image_base_Inpaint.jpg b/LanPaint/example_workflows/Z_image_base_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b5c62bdbdcd1d25b6ef1ce6281ba038d197e4061 --- /dev/null +++ b/LanPaint/example_workflows/Z_image_base_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0a7c8946f52c092a10320542d316285f3fbe6c469be0084d636f94c7aad5383 +size 1197078 diff --git a/LanPaint/example_workflows/Z_image_base_Inpaint.json b/LanPaint/example_workflows/Z_image_base_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..01295e164d929ed04e993afb8ba872e5366a38e1 --- /dev/null +++ b/LanPaint/example_workflows/Z_image_base_Inpaint.json @@ -0,0 +1,1548 @@ +{ + "id": "9ae6082b-c7f4-433c-9971-7a8f65a3ea65", + "revision": 0, + "last_node_id": 79, + "last_link_id": 106, + "nodes": [ + { + "id": 40, + "type": "VAELoader", + "pos": [ + 131.0846434356025, + 576.6065838350738 + ], + "size": [ + 269.77864583333337, + 67.82552083333334 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 39, + 70, + 86, + 87 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "models": [ + { + "name": "ae.safetensors", + "url": "https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/vae/ae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "ae.safetensors" + ] + }, + { + "id": 41, + "type": "EmptySD3LatentImage", + "pos": [ + 130.26337981400314, + 705.2539588236402 + ], + "size": [ + 259.77864583333337, + 124.4921875 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "Node name for S&R": "EmptySD3LatentImage", + "cnr_id": "comfy-core", + "ver": "0.3.64", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 43, + "type": "VAEDecode", + "pos": [ + 1240.000653308445, + 139.99990859818013 + ], + "size": [ + 209.77864583333334, + 59.51822916666667 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 104 + }, + { + "name": "vae", + "type": "VAE", + "link": 39 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 73, + 93 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.3.64", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [] + }, + { + "id": 48, + "type": "LoraLoaderModelOnly", + "pos": [ + 459.99957646933433, + 109.99990579628181 + ], + "size": [ + 369.7916666666667, + 96.15885416666667 + ], + "flags": {}, + "order": 8, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 54 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 60 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.3.75", + "models": [ + { + "name": "pixel_art_style_z_image_turbo.safetensors", + "url": "https://huggingface.co/tarn59/pixel_art_style_lora_z_image_turbo/resolve/main/pixel_art_style_z_image_turbo.safetensors", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "pixel_art_style_z_image_turbo.safetensors", + 1 + ] + }, + { + "id": 35, + "type": "MarkdownNote", + "pos": [ + -390.00009654097465, + 239.99994184235885 + ], + "size": [ + 489.77864583333337, + 617.3828125 + ], + "flags": { + "collapsed": false + }, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model link", + "properties": {}, + "widgets_values": [ + "## Report workflow issue\n\nIf you found any issues when running this workflow, [report template issue here](https://github.com/Comfy-Org/workflow_templates/issues)\n\n\n## Model links\n\n**text_encoders**\n\n- [qwen_3_4b.safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/text_encoders/qwen_3_4b.safetensors)\n\n**loras**\n\n- [pixel_art_style_z_image_turbo.safetensors](https://huggingface.co/tarn59/pixel_art_style_lora_z_image_turbo/resolve/main/pixel_art_style_z_image_turbo.safetensors)\n\n**diffusion_models**\n\n- [z_image_turbo_bf16.safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/diffusion_models/z_image_turbo_bf16.safetensors)\n\n**vae**\n\n- [ae.safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/vae/ae.safetensors)\n\n\nModel Storage Location\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ └── qwen_3_4b.safetensors\n│ ├── 📂 loras/\n│ │ └── pixel_art_style_z_image_turbo.safetensors\n│ ├── 📂 diffusion_models/\n│ │ └── z_image_turbo_bf16.safetensors\n│ └── 📂 vae/\n│ └── ae.safetensors\n```\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 58, + "type": "VAEEncode", + "pos": [ + 1416.1429111903612, + 1387.521796150247 + ], + "size": [ + 187.5, + 59.51822916666667 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 90 + }, + { + "name": "vae", + "type": "VAE", + "link": 70 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "VAEEncode", + "cnr_id": "comfy-core", + "ver": "0.3.59", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 59, + "type": "SetLatentNoiseMask", + "pos": [ + 1805.8745237820265, + 1362.6670257146513 + ], + "size": [ + 187.5, + 59.51822916666667 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 67 + }, + { + "name": "mask", + "type": "MASK", + "link": 91 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 103 + ] + } + ], + "properties": { + "Node name for S&R": "SetLatentNoiseMask", + "cnr_id": "comfy-core", + "ver": "0.3.59", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 60, + "type": "LanPaint_MaskBlend", + "pos": [ + 2285.630742528444, + 1036.4610004736458 + ], + "size": [ + 210, + 107.83854166666667 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 94 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 73 + }, + { + "name": "mask", + "type": "MASK", + "link": 95 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 72 + ] + } + ], + "properties": { + "Node name for S&R": "LanPaint_MaskBlend", + "cnr_id": "LanPaint", + "ver": "4d3d5d17f0105b673df92da5b084cce567c9c712" + }, + "widgets_values": [ + 9 + ] + }, + { + "id": 61, + "type": "SaveImage", + "pos": [ + 2759.9685719809763, + 991.0132526957497 + ], + "size": [ + 1086.7317708333335, + 1170.0130208333335 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 72 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 62, + "type": "VAEDecode", + "pos": [ + 756.1992164021267, + 1691.7960954209382 + ], + "size": [ + 209.921875, + 59.51822916666667 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 76 + }, + { + "name": "vae", + "type": "VAE", + "link": 87 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.3.23" + }, + "widgets_values": [] + }, + { + "id": 64, + "type": "MaskToImage", + "pos": [ + 1065.0878525980884, + 1566.3928981929303 + ], + "size": [ + 187.5, + 39.51822916666667 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 89 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage", + "cnr_id": "comfy-core", + "ver": "0.3.51" + }, + "widgets_values": [] + }, + { + "id": 67, + "type": "VAEEncode", + "pos": [ + 770.9260250187167, + 1599.2336900135522 + ], + "size": [ + 187.5, + 59.51822916666667 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 85 + }, + { + "name": "vae", + "type": "VAE", + "link": 86 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 76 + ] + } + ], + "properties": { + "Node name for S&R": "VAEEncode", + "cnr_id": "comfy-core", + "ver": "0.3.50", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "widget_ue_connectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 68, + "type": "ImageScale", + "pos": [ + 1059.1640261748782, + 1655.6404162754113 + ], + "size": [ + 269.921875, + 152.8125 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 81 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 82 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 83 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "ImageScale", + "cnr_id": "comfy-core", + "ver": "0.3.52" + }, + "widgets_values": [ + "nearest-exact", + 512, + 512, + "center" + ] + }, + { + "id": 69, + "type": "GetImageSize", + "pos": [ + 768.3583929275752, + 1791.1471927140292 + ], + "size": [ + 187.5, + 133.4765625 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 84 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 77, + 82 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 78, + 83 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.3.52" + }, + "widgets_values": [ + "width: 1024, height: 1024\n batch size: 1" + ] + }, + { + "id": 63, + "type": "ImageScale", + "pos": [ + 1068.1979260124726, + 1379.5306113893253 + ], + "size": [ + 269.921875, + 152.8125 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 88 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 77 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 78 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 90, + 94 + ] + } + ], + "properties": { + "Node name for S&R": "ImageScale", + "cnr_id": "comfy-core", + "ver": "0.3.52" + }, + "widgets_values": [ + "area", + 512, + 512, + "center" + ] + }, + { + "id": 65, + "type": "ImageToMask", + "pos": [ + 1060.4634757752433, + 1829.122961862717 + ], + "size": [ + 269.921875, + 67.82552083333334 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 79 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 91, + 95 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToMask", + "cnr_id": "comfy-core", + "ver": "0.3.51" + }, + "widgets_values": [ + "red" + ] + }, + { + "id": 72, + "type": "Note", + "pos": [ + 1072.3258089884341, + 1251.3163769049763 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Reshape image to suit VAE " + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 46, + "type": "UNETLoader", + "pos": [ + 130.8803835176007, + 238.85087590367888 + ], + "size": [ + 269.77864583333337, + 96.15885416666667 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 54 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "models": [ + { + "name": "z_image_turbo_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/diffusion_models/z_image_turbo_bf16.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "z_image_bf16.safetensors", + "default" + ] + }, + { + "id": 57, + "type": "LoadImage", + "pos": [ + 6.077161599140862, + 923.8378757225194 + ], + "size": [ + 273.88020833333337, + 546.8098958333334 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 85, + 88 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 89 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "cnr_id": "comfy-core", + "ver": "0.3.59", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.1", + "input_ue_unconnectable": {} + }, + "image": "clipspace/clipspace-painted-masked-1769703696385.png [input]" + }, + "widgets_values": [ + "clipspace/clipspace-painted-masked-1769703696385.png [input]", + "image" + ] + }, + { + "id": 39, + "type": "CLIPLoader", + "pos": [ + 130.26337981400314, + 405.2542176588747 + ], + "size": [ + 269.77864583333337, + 124.4921875 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 44, + 98 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "models": [ + { + "name": "qwen_3_4b.safetensors", + "url": "https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/text_encoders/qwen_3_4b.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "lumina2", + "default" + ] + }, + { + "id": 75, + "type": "CLIPTextEncode", + "pos": [ + 448.7872367553159, + 690.9253024127064 + ], + "size": [ + 390, + 125 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 98 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 102 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 73, + "type": "PreviewImage", + "pos": [ + 1828.9318713728562, + 114.5680075231259 + ], + "size": [ + 187.5, + 246 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 93 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.3.76" + }, + "widgets_values": [] + }, + { + "id": 47, + "type": "ModelSamplingAuraFlow", + "pos": [ + 920.7037987240392, + 248.48064332359456 + ], + "size": [ + 309.89583333333337, + 67.82552083333334 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 60 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 100 + ] + } + ], + "properties": { + "Node name for S&R": "ModelSamplingAuraFlow", + "cnr_id": "comfy-core", + "ver": "0.3.64", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 56, + "type": "LanPaint_KSampler", + "pos": [ + 1869.2934574860883, + 468.13621723594605 + ], + "size": [ + 372.64694675564056, + 406.22742984745275 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LanPaint_KSampler", + "cnr_id": "LanPaint", + "ver": "6109df6591a4cf2bc9d3b113d03f7297fa9248e9" + }, + "widgets_values": [ + 1011254864048366, + "randomize", + 20, + 4, + "euler", + "simple", + 1, + 5, + "Prompt First", + "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", + "🖼️ Image Inpainting" + ] + }, + { + "id": 45, + "type": "CLIPTextEncode", + "pos": [ + 450.2633140828448, + 275.25446846701436 + ], + "size": [ + 409.77864583333337, + 374.77864583333337 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 44 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 101 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.73", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "A fashion photography work full of surreal romanticism, using a low-angle upward shooting composition, with a clear light blue sky as the background, and the visual focus concentrated on the fantasy blue vegetation and the cute gaint panda walking through it.\n\nThe vegetation in the picture is processed into varying shades of blue, from light ice blue to deep cobalt blue. The textures of the leaves and branches are delicate and realistic. The warm brown tree trunks form a sharp contrast with the cool blue leaves, resembling a dreamy forest from another world. A cute gaint panda walks slowly on the sand. The warm tones of the dress echo with the surrounding cool blue vegetation. The noon sun casts clear shadows on the sand, enhancing the sense of space and reality in the picture.\n\nThe entire scene, with its clean and transparent colors and fantasy settings, not only exudes the vastness of the natural wilderness but also presents a quiet and poetic high-fashion sense due to the surreal vegetation." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 76, + "type": "LanPaint_KSamplerAdvanced", + "pos": [ + 1234.9129727307572, + 465.17562131822933 + ], + "size": [ + 400, + 580 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 100 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 101 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 102 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 103 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 104 + ] + } + ], + "properties": { + "Node name for S&R": "LanPaint_KSamplerAdvanced" + }, + "widgets_values": [ + "enable", + 0, + "fixed", + 20, + 4, + "euler", + "simple", + 0, + 10000, + "disable", + 3, + 8, + 0.05, + 1, + 25, + "Image First", + 1, + "LanPaint KSampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", + "🖼️ Image Inpainting" + ] + } + ], + "links": [ + [ + 39, + 40, + 0, + 43, + 1, + "VAE" + ], + [ + 44, + 39, + 0, + 45, + 0, + "CLIP" + ], + [ + 54, + 46, + 0, + 48, + 0, + "MODEL" + ], + [ + 60, + 48, + 0, + 47, + 0, + "MODEL" + ], + [ + 67, + 58, + 0, + 59, + 0, + "LATENT" + ], + [ + 70, + 40, + 0, + 58, + 1, + "VAE" + ], + [ + 72, + 60, + 0, + 61, + 0, + "IMAGE" + ], + [ + 73, + 43, + 0, + 60, + 1, + "IMAGE" + ], + [ + 76, + 67, + 0, + 62, + 0, + "LATENT" + ], + [ + 77, + 69, + 0, + 63, + 1, + "INT" + ], + [ + 78, + 69, + 1, + 63, + 2, + "INT" + ], + [ + 79, + 68, + 0, + 65, + 0, + "IMAGE" + ], + [ + 81, + 64, + 0, + 68, + 0, + "IMAGE" + ], + [ + 82, + 69, + 0, + 68, + 1, + "INT" + ], + [ + 83, + 69, + 1, + 68, + 2, + "INT" + ], + [ + 84, + 62, + 0, + 69, + 0, + "IMAGE" + ], + [ + 85, + 57, + 0, + 67, + 0, + "IMAGE" + ], + [ + 86, + 40, + 0, + 67, + 1, + "VAE" + ], + [ + 87, + 40, + 0, + 62, + 1, + "VAE" + ], + [ + 88, + 57, + 0, + 63, + 0, + "IMAGE" + ], + [ + 89, + 57, + 1, + 64, + 0, + "MASK" + ], + [ + 90, + 63, + 0, + 58, + 0, + "IMAGE" + ], + [ + 91, + 65, + 0, + 59, + 1, + "MASK" + ], + [ + 93, + 43, + 0, + 73, + 0, + "IMAGE" + ], + [ + 94, + 63, + 0, + 60, + 0, + "IMAGE" + ], + [ + 95, + 65, + 0, + 60, + 2, + "MASK" + ], + [ + 98, + 39, + 0, + 75, + 0, + "CLIP" + ], + [ + 100, + 47, + 0, + 76, + 0, + "MODEL" + ], + [ + 101, + 45, + 0, + 76, + 1, + "CONDITIONING" + ], + [ + 102, + 75, + 0, + 76, + 2, + "CONDITIONING" + ], + [ + 103, + 59, + 0, + 76, + 3, + "LATENT" + ], + [ + 104, + 76, + 0, + 43, + 0, + "LATENT" + ] + ], + "groups": [ + { + "id": 2, + "title": "Step2 - Image size", + "bounding": [ + 120, + 640, + 290, + 200 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Step3 - Prompt", + "bounding": [ + 430, + 210, + 450, + 540 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Step1 - Load models", + "bounding": [ + 120, + 210, + 290, + 413.6 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "Ctrl-B to enable LoRA input", + "bounding": [ + 430, + 40, + 440, + 160 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.2673486306413776, + "offset": [ + 853.4341376683553, + 289.09247282952845 + ] + }, + "frontendVersion": "1.37.11", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "workflowRendererVersion": "LG" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/wan2_2_T2I_Inpaint.jpg b/LanPaint/example_workflows/wan2_2_T2I_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..150b241953e996fc898f7a2624835234de712f11 --- /dev/null +++ b/LanPaint/example_workflows/wan2_2_T2I_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d09ce42506d6ebdbbd43a3105f050091f93c439a4e10c76b22c8e391c18a7b35 +size 699393 diff --git a/LanPaint/example_workflows/wan2_2_T2I_Inpaint.json b/LanPaint/example_workflows/wan2_2_T2I_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..b250784f19f335d99820f7fe4ad62f91bf834058 --- /dev/null +++ b/LanPaint/example_workflows/wan2_2_T2I_Inpaint.json @@ -0,0 +1,1330 @@ +{ + "id": "26bf2769-956f-4cee-b5f6-a57a366c9101", + "revision": 0, + "last_node_id": 83, + "last_link_id": 180, + "nodes": [ + { + "id": 56, + "type": "UNETLoader", + "pos": [ + 34.70310592651367, + 57.175601959228516 + ], + "size": [ + 346.7470703125, + 82 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 112 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 54, + "type": "ModelSamplingSD3", + "pos": [ + 640, + -70 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 110 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 150 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "ModelSamplingSD3", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 55, + "type": "ModelSamplingSD3", + "pos": [ + 640, + 60 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 112 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 159 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "ModelSamplingSD3", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 62, + "type": "MarkdownNote", + "pos": [ + -480, + -120 + ], + "size": [ + 480, + 350 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model Links", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "[Tutorial](https://docs.comfy.org/tutorials/video/wan/wan2_2\n) | [教程](https://docs.comfy.org/zh-CN/tutorials/video/wan/wan2_2\n)\n\n**Diffusion Model**\n- [wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors)\n- [wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors)\n\n**VAE**\n- [wan_2.1_vae.safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/vae/wan_2.1_vae.safetensors)\n\n**Text Encoder** \n- [umt5_xxl_fp8_e4m3fn_scaled.safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors)\n\n\nFile save location\n\n```\nComfyUI/\n├───📂 models/\n│ ├───📂 diffusion_models/\n│ │ ├─── wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors\n│ │ └─── wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors\n│ ├───📂 text_encoders/\n│ │ └─── umt5_xxl_fp8_e4m3fn_scaled.safetensors \n│ └───📂 vae/\n│ └── wan_2.1_vae.safetensors\n```\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 415.27801513671875, + 562.8309936523438 + ], + "size": [ + 425.27801513671875, + 180.6060791015625 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 75 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 152, + 161 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "CLIPTextEncode", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 59, + "type": "EmptyHunyuanLatentVideo", + "pos": [ + 50, + 530 + ], + "size": [ + 315, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "EmptyHunyuanLatentVideo", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 768, + 1344, + 1, + 1 + ] + }, + { + "id": 37, + "type": "UNETLoader", + "pos": [ + 34.225364685058594, + -71.77911376953125 + ], + "size": [ + 346.7470703125, + 82 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 110 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 38, + "type": "CLIPLoader", + "pos": [ + 38.86457061767578, + 189.26513671875 + ], + "size": [ + 346.391845703125, + 106 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 74, + 75 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "directory": "text_encoders" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 67, + "type": "PrimitiveNode", + "pos": [ + 874.0289916992188, + 28.25248908996582 + ], + "size": [ + 312.7867736816406, + 88.04627990722656 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "widget": { + "name": "steps" + }, + "links": [ + 154, + 163 + ] + } + ], + "title": "Steps", + "properties": { + "Run widget replace on values": false, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 20, + "fixed" + ] + }, + { + "id": 68, + "type": "PrimitiveNode", + "pos": [ + 881.3255615234375, + 167.12957763671875 + ], + "size": [ + 299.6272277832031, + 82 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "widget": { + "name": "end_at_step" + }, + "links": [ + 157, + 166 + ] + } + ], + "title": "Switch_step", + "properties": { + "Run widget replace on values": false, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 3, + "fixed" + ] + }, + { + "id": 66, + "type": "PrimitiveNode", + "pos": [ + 869.0523071289062, + 458.133544921875 + ], + "size": [ + 302.1168212890625, + 106 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "COMBO", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "links": [ + 156, + 165 + ] + } + ], + "title": "Scheduler", + "properties": { + "Run widget replace on values": false, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sgm_uniform", + "fixed", + "" + ] + }, + { + "id": 71, + "type": "VAEEncode", + "pos": [ + 386.7767333984375, + 924.7274780273438 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 143 + }, + { + "name": "vae", + "type": "VAE", + "link": 146 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 148 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [] + }, + { + "id": 65, + "type": "PrimitiveNode", + "pos": [ + 871.877197265625, + 301.2847595214844 + ], + "size": [ + 304.6064453125, + 106 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "COMBO", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "links": [ + 155, + 164 + ] + } + ], + "title": "Sampler", + "properties": { + "Run widget replace on values": false, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "euler", + "fixed", + "" + ] + }, + { + "id": 39, + "type": "VAELoader", + "pos": [ + 40, + 350 + ], + "size": [ + 344.731689453125, + 59.98149108886719 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 76, + 146 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "wan_2.1_vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/vae/wan_2.1_vae.safetensors", + "directory": "vae" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "wan_2.1_vae.safetensors" + ] + }, + { + "id": 70, + "type": "SaveImage", + "pos": [ + 1818.5047607421875, + -71.57525634765625 + ], + "size": [ + 270, + 270 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 142 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 74, + "type": "SetLatentNoiseMask", + "pos": [ + 469.00274658203125, + 822.289306640625 + ], + "size": [ + 180.74765014648438, + 46 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 148 + }, + { + "name": "mask", + "type": "MASK", + "link": 149 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 167 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 75, + "type": "LanPaint_KSamplerAdvanced", + "pos": [ + 1271.481201171875, + -503.987548828125 + ], + "size": [ + 400, + 764 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 150 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 151 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 152 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 167 + }, + { + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 154 + }, + { + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": 155 + }, + { + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": 156 + }, + { + "name": "end_at_step", + "type": "INT", + "widget": { + "name": "end_at_step" + }, + "link": 157 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 172 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "1b50a90424318a457ff32a1fd5ee8a441b768b45", + "Node name for S&R": "LanPaint_KSamplerAdvanced" + }, + "widgets_values": [ + "enable", + 310185466124726, + "randomize", + 20, + 2, + "euler", + "sgm_uniform", + 0, + 3, + "disable", + 5, + 16, + 0.15, + 1, + 15, + "Image First", + 1, + "LanPaint KSampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1836.6907958984375, + -249.27249145507812 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 162 + }, + { + "name": "vae", + "type": "VAE", + "link": 76 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 142 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "VAEDecode", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 63, + "type": "Note", + "pos": [ + 20, + 720 + ], + "size": [ + 370, + 120 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Video Size", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "By default, we set the video to a smaller size for users with low VRAM. If you have enough VRAM, you can change the size" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 76, + "type": "LanPaint_KSamplerAdvanced", + "pos": [ + 1276.2220458984375, + 325.4043884277344 + ], + "size": [ + 400, + 764 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 159 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 160 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 161 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 172 + }, + { + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 163 + }, + { + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": 164 + }, + { + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": 165 + }, + { + "name": "start_at_step", + "type": "INT", + "widget": { + "name": "start_at_step" + }, + "link": 166 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 162 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "1b50a90424318a457ff32a1fd5ee8a441b768b45", + "Node name for S&R": "LanPaint_KSamplerAdvanced" + }, + "widgets_values": [ + "enable", + 542259685031634, + "randomize", + 20, + 2, + "euler", + "sgm_uniform", + 3, + 10000, + "disable", + 5, + 16, + 0.15, + 1, + 15, + "Image First", + 1, + "LanPaint KSampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + }, + { + "id": 82, + "type": "Note", + "pos": [ + 1722.8084716796875, + -428.4427185058594 + ], + "size": [ + 370, + 120 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Acceleration tip", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Reduce LanPaint NumSteps for faster generation!" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 83, + "type": "Note", + "pos": [ + 1736.30029296875, + 592.1900024414062 + ], + "size": [ + 370, + 120 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Acceleration tip", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Increase LanPaint Early Stop for faster generation!" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 420, + 280 + ], + "size": [ + 425.4649963378906, + 225.1785888671875 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 74 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 151, + 160 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "CLIPTextEncode", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Close up Portrait of a futuristic cowgirl wearing jeans rides a mechanical horse across a vast cyberpunk desert. The dunes shimmer with holographic projections, glowing in blues and purples. cacti and two massive moons hang low on the horizon, illuminating the dusty air. Cyberpunk western fusion, cinematic, 8K, highly detailed, atmospheric perspective, Blade Runner style." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 69, + "type": "LoadImage", + "pos": [ + 38.325469970703125, + 964.1947021484375 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 143 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 149 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "Masked_Load_Me_in_Loader (8).png", + "image" + ] + } + ], + "links": [ + [ + 74, + 38, + 0, + 6, + 0, + "CLIP" + ], + [ + 75, + 38, + 0, + 7, + 0, + "CLIP" + ], + [ + 76, + 39, + 0, + 8, + 1, + "VAE" + ], + [ + 110, + 37, + 0, + 54, + 0, + "MODEL" + ], + [ + 112, + 56, + 0, + 55, + 0, + "MODEL" + ], + [ + 142, + 8, + 0, + 70, + 0, + "IMAGE" + ], + [ + 143, + 69, + 0, + 71, + 0, + "IMAGE" + ], + [ + 146, + 39, + 0, + 71, + 1, + "VAE" + ], + [ + 148, + 71, + 0, + 74, + 0, + "LATENT" + ], + [ + 149, + 69, + 1, + 74, + 1, + "MASK" + ], + [ + 150, + 54, + 0, + 75, + 0, + "MODEL" + ], + [ + 151, + 6, + 0, + 75, + 1, + "CONDITIONING" + ], + [ + 152, + 7, + 0, + 75, + 2, + "CONDITIONING" + ], + [ + 154, + 67, + 0, + 75, + 4, + "INT" + ], + [ + 155, + 65, + 0, + 75, + 5, + "COMBO" + ], + [ + 156, + 66, + 0, + 75, + 6, + "COMBO" + ], + [ + 157, + 68, + 0, + 75, + 7, + "INT" + ], + [ + 159, + 55, + 0, + 76, + 0, + "MODEL" + ], + [ + 160, + 6, + 0, + 76, + 1, + "CONDITIONING" + ], + [ + 161, + 7, + 0, + 76, + 2, + "CONDITIONING" + ], + [ + 162, + 76, + 0, + 8, + 0, + "LATENT" + ], + [ + 163, + 67, + 0, + 76, + 4, + "INT" + ], + [ + 164, + 65, + 0, + 76, + 5, + "COMBO" + ], + [ + 165, + 66, + 0, + 76, + 6, + "COMBO" + ], + [ + 166, + 68, + 0, + 76, + 7, + "INT" + ], + [ + 167, + 74, + 0, + 75, + 3, + "LATENT" + ], + [ + 172, + 75, + 0, + 76, + 3, + "LATENT" + ] + ], + "groups": [ + { + "id": 1, + "title": "Step1 - Load models", + "bounding": [ + 20, + -150, + 371.0310363769531, + 571.3974609375 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Step 1: Image size", + "bounding": [ + 20, + 450, + 367.5572814941406, + 222.82713317871094 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Step3 Prompt", + "bounding": [ + 410, + 210, + 445.27801513671875, + 464.2060852050781 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.4241196229884343, + "offset": [ + 228.44262760495175, + 461.85293161095177 + ] + }, + "frontendVersion": "1.25.11", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "ue_links": [], + "links_added_by_ue": [] + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/example_workflows/wan2_2_T2I_Partial_Inpaint.jpg b/LanPaint/example_workflows/wan2_2_T2I_Partial_Inpaint.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc7cd4e60676c65855791b282a7f588dd3aa6ae2 --- /dev/null +++ b/LanPaint/example_workflows/wan2_2_T2I_Partial_Inpaint.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3305c24fa1e2d426f36103c4a8be9581944910c651f3d6bbcadf36419d9310 +size 703660 diff --git a/LanPaint/example_workflows/wan2_2_T2I_Partial_Inpaint.json b/LanPaint/example_workflows/wan2_2_T2I_Partial_Inpaint.json new file mode 100644 index 0000000000000000000000000000000000000000..9f0d3e3af1867582d1f2d8404d6ba68b50c8c811 --- /dev/null +++ b/LanPaint/example_workflows/wan2_2_T2I_Partial_Inpaint.json @@ -0,0 +1,1128 @@ +{ + "id": "26bf2769-956f-4cee-b5f6-a57a366c9101", + "revision": 0, + "last_node_id": 84, + "last_link_id": 181, + "nodes": [ + { + "id": 56, + "type": "UNETLoader", + "pos": [ + 34.70310592651367, + 57.175601959228516 + ], + "size": [ + 346.7470703125, + 82 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 112 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 54, + "type": "ModelSamplingSD3", + "pos": [ + 640, + -70 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 110 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "ModelSamplingSD3", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 55, + "type": "ModelSamplingSD3", + "pos": [ + 640, + 60 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 112 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 159 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "ModelSamplingSD3", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 62, + "type": "MarkdownNote", + "pos": [ + -480, + -120 + ], + "size": [ + 480, + 350 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model Links", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "[Tutorial](https://docs.comfy.org/tutorials/video/wan/wan2_2\n) | [教程](https://docs.comfy.org/zh-CN/tutorials/video/wan/wan2_2\n)\n\n**Diffusion Model**\n- [wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors)\n- [wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors)\n\n**VAE**\n- [wan_2.1_vae.safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/vae/wan_2.1_vae.safetensors)\n\n**Text Encoder** \n- [umt5_xxl_fp8_e4m3fn_scaled.safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors)\n\n\nFile save location\n\n```\nComfyUI/\n├───📂 models/\n│ ├───📂 diffusion_models/\n│ │ ├─── wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors\n│ │ └─── wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors\n│ ├───📂 text_encoders/\n│ │ └─── umt5_xxl_fp8_e4m3fn_scaled.safetensors \n│ └───📂 vae/\n│ └── wan_2.1_vae.safetensors\n```\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 415.27801513671875, + 562.8309936523438 + ], + "size": [ + 425.27801513671875, + 180.6060791015625 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 75 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 161 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "CLIPTextEncode", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 59, + "type": "EmptyHunyuanLatentVideo", + "pos": [ + 50, + 530 + ], + "size": [ + 315, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "EmptyHunyuanLatentVideo", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 768, + 1344, + 1, + 1 + ] + }, + { + "id": 37, + "type": "UNETLoader", + "pos": [ + 34.225364685058594, + -71.77911376953125 + ], + "size": [ + 346.7470703125, + 82 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 110 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/diffusion_models/wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 38, + "type": "CLIPLoader", + "pos": [ + 38.86457061767578, + 189.26513671875 + ], + "size": [ + 346.391845703125, + 106 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 74, + 75 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "directory": "text_encoders" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 66, + "type": "PrimitiveNode", + "pos": [ + 869.0523071289062, + 458.133544921875 + ], + "size": [ + 302.1168212890625, + 106 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "COMBO", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "links": [ + 165 + ] + } + ], + "title": "Scheduler", + "properties": { + "Run widget replace on values": false, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sgm_uniform", + "fixed", + "" + ] + }, + { + "id": 71, + "type": "VAEEncode", + "pos": [ + 386.7767333984375, + 924.7274780273438 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 143 + }, + { + "name": "vae", + "type": "VAE", + "link": 146 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 148 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [] + }, + { + "id": 39, + "type": "VAELoader", + "pos": [ + 40, + 350 + ], + "size": [ + 344.731689453125, + 59.98149108886719 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 76, + 146 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "wan_2.1_vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/vae/wan_2.1_vae.safetensors", + "directory": "vae" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "wan_2.1_vae.safetensors" + ] + }, + { + "id": 70, + "type": "SaveImage", + "pos": [ + 1818.5047607421875, + -71.57525634765625 + ], + "size": [ + 270, + 270 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 142 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 74, + "type": "SetLatentNoiseMask", + "pos": [ + 469.00274658203125, + 822.289306640625 + ], + "size": [ + 180.74765014648438, + 46 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 148 + }, + { + "name": "mask", + "type": "MASK", + "link": 149 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 181 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "SetLatentNoiseMask" + }, + "widgets_values": [] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1836.6907958984375, + -249.27249145507812 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 162 + }, + { + "name": "vae", + "type": "VAE", + "link": 76 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 142 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "VAEDecode", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 63, + "type": "Note", + "pos": [ + 20, + 720 + ], + "size": [ + 370, + 120 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Video Size", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "By default, we set the video to a smaller size for users with low VRAM. If you have enough VRAM, you can change the size" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 82, + "type": "Note", + "pos": [ + 1722.8084716796875, + -428.4427185058594 + ], + "size": [ + 370, + 120 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Acceleration tip", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Reduce LanPaint NumSteps for faster generation!" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 420, + 280 + ], + "size": [ + 425.4649963378906, + 225.1785888671875 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 74 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 160 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.45", + "Node name for S&R": "CLIPTextEncode", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Close up Portrait of a futuristic cowgirl wearing jeans rides a mechanical horse across a vast cyberpunk desert. The dunes shimmer with holographic projections, glowing in blues and purples. cacti and two massive moons hang low on the horizon, illuminating the dusty air. Cyberpunk western fusion, cinematic, 8K, highly detailed, atmospheric perspective, Blade Runner style." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 69, + "type": "LoadImage", + "pos": [ + 38.325469970703125, + 964.1947021484375 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 143 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 149 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.59", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "Masked_Load_Me_in_Loader (8).png", + "image" + ] + }, + { + "id": 65, + "type": "PrimitiveNode", + "pos": [ + 871.877197265625, + 301.2847595214844 + ], + "size": [ + 304.6064453125, + 106 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "COMBO", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "links": [ + 164 + ] + } + ], + "title": "Sampler", + "properties": { + "Run widget replace on values": false, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "euler", + "fixed", + "" + ] + }, + { + "id": 67, + "type": "PrimitiveNode", + "pos": [ + 874.0289916992188, + 28.25248908996582 + ], + "size": [ + 312.7867736816406, + 88.04627990722656 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "widget": { + "name": "steps" + }, + "links": [ + 163 + ] + } + ], + "title": "Steps", + "properties": { + "Run widget replace on values": false, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 20, + "fixed" + ] + }, + { + "id": 83, + "type": "Note", + "pos": [ + 1736.30029296875, + 592.1900024414062 + ], + "size": [ + 370, + 120 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Acceleration tip", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Increase LanPaint Early Stop for faster generation!" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 84, + "type": "Note", + "pos": [ + 1742.7166748046875, + 415.5233459472656 + ], + "size": [ + 370, + 120 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Partial Inpaint Strength", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Change Start_at_step to control how much you want inpainting to keep original picture's content" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 76, + "type": "LanPaint_KSamplerAdvanced", + "pos": [ + 1276.2220458984375, + 325.4043884277344 + ], + "size": [ + 400, + 764 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 159 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 160 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 161 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 181 + }, + { + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 163 + }, + { + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": 164 + }, + { + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": 165 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 162 + ] + } + ], + "properties": { + "cnr_id": "LanPaint", + "ver": "1b50a90424318a457ff32a1fd5ee8a441b768b45", + "Node name for S&R": "LanPaint_KSamplerAdvanced" + }, + "widgets_values": [ + "enable", + 542259685031634, + "fixed", + 20, + 2, + "euler", + "sgm_uniform", + 7, + 10000, + "disable", + 5, + 16, + 0.15, + 1, + 15, + "Image First", + 1, + "LanPaint KSampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!" + ] + } + ], + "links": [ + [ + 74, + 38, + 0, + 6, + 0, + "CLIP" + ], + [ + 75, + 38, + 0, + 7, + 0, + "CLIP" + ], + [ + 76, + 39, + 0, + 8, + 1, + "VAE" + ], + [ + 110, + 37, + 0, + 54, + 0, + "MODEL" + ], + [ + 112, + 56, + 0, + 55, + 0, + "MODEL" + ], + [ + 142, + 8, + 0, + 70, + 0, + "IMAGE" + ], + [ + 143, + 69, + 0, + 71, + 0, + "IMAGE" + ], + [ + 146, + 39, + 0, + 71, + 1, + "VAE" + ], + [ + 148, + 71, + 0, + 74, + 0, + "LATENT" + ], + [ + 149, + 69, + 1, + 74, + 1, + "MASK" + ], + [ + 159, + 55, + 0, + 76, + 0, + "MODEL" + ], + [ + 160, + 6, + 0, + 76, + 1, + "CONDITIONING" + ], + [ + 161, + 7, + 0, + 76, + 2, + "CONDITIONING" + ], + [ + 162, + 76, + 0, + 8, + 0, + "LATENT" + ], + [ + 163, + 67, + 0, + 76, + 4, + "INT" + ], + [ + 164, + 65, + 0, + 76, + 5, + "COMBO" + ], + [ + 165, + 66, + 0, + 76, + 6, + "COMBO" + ], + [ + 181, + 74, + 0, + 76, + 3, + "LATENT" + ] + ], + "groups": [ + { + "id": 1, + "title": "Step1 - Load models", + "bounding": [ + 20, + -150, + 371.0310363769531, + 571.3974609375 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Step 1: Image size", + "bounding": [ + 20, + 450, + 367.5572814941406, + 222.82713317871094 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Step3 Prompt", + "bounding": [ + 410, + 210, + 445.27801513671875, + 464.2060852050781 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.48236739945295026, + "offset": [ + 442.47629449189617, + -54.01599879799346 + ] + }, + "frontendVersion": "1.25.11", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "ue_links": [], + "links_added_by_ue": [] + }, + "version": 0.4 +} \ No newline at end of file diff --git a/LanPaint/examples/Example_1/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_1/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..a52f36b9d6fcd269f45e819d1b5353402e60853e --- /dev/null +++ b/LanPaint/examples/Example_1/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12871ed1f7240652ea234dbb2b2ee05639d239b373d316c45de7e209de472bd +size 1641900 diff --git a/LanPaint/examples/Example_1/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_1/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..87c6a39a2c6b0935eb8486e89a726dd7f13f0bec --- /dev/null +++ b/LanPaint/examples/Example_1/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fb2cf92b2d13e70914750775cd665ebf6925177aea57d2c7c11053de629323c +size 1816744 diff --git a/LanPaint/examples/Example_1/Original_No_Mask.png b/LanPaint/examples/Example_1/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..dc3c1eee0df5542279a98d89cb5248319094224b --- /dev/null +++ b/LanPaint/examples/Example_1/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01fb3adef12243ad4995de22b0b18e183f1a54c22d4bde91cd06d0656f2b393a +size 1584211 diff --git a/LanPaint/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..50d9d4980015a2c6ddda7f4db3acbfb96895bda4 --- /dev/null +++ b/LanPaint/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1da4fc96bd3b1f8fade0b2ae5c1048c05b133cab7fa280beb3441df2f522bf7 +size 1651846 diff --git a/LanPaint/examples/Example_10/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_10/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..27febd79fc064d2f0ec3bcb9510b20b457724651 --- /dev/null +++ b/LanPaint/examples/Example_10/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7963487802b3ff300012e253730de79c6669fde8467f71223c4490a9ea6bee3 +size 1333530 diff --git a/LanPaint/examples/Example_11/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_11/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..0ee79f0dbf2ee6311532c24c913362e2c65905a5 --- /dev/null +++ b/LanPaint/examples/Example_11/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e09023e36178ffe2cc808533832d7dc44866ad81bfe57c03f07ec003b8556cc +size 2658600 diff --git a/LanPaint/examples/Example_11/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_11/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..357153d6b455b1afd9db0f650f7ec9dac23db487 --- /dev/null +++ b/LanPaint/examples/Example_11/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64f301dee695b08e2c248de8344f66aa5f01de7cc1c99b7d786410a9ac1ae01e +size 2992000 diff --git a/LanPaint/examples/Example_12/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_12/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..7cd480c7b71770628d2e282e0dd0f695457374b2 --- /dev/null +++ b/LanPaint/examples/Example_12/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b527f6bf1f7d218580928c62fed68af97dc272a8e726931da5acd5a6bdacc048 +size 1796016 diff --git a/LanPaint/examples/Example_12/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_12/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..ac316e51e9b4adc5f125c63240cd6b082a49106c --- /dev/null +++ b/LanPaint/examples/Example_12/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7be87e02c72d9701f6c86bbfff31dcd46109d58e387c0297ddd2099c13c51a94 +size 850264 diff --git a/LanPaint/examples/Example_13/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_13/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..8d4862f9cc906c42cfb6eba1e80f49307e60a7fb --- /dev/null +++ b/LanPaint/examples/Example_13/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:667a6a059899c5a75ddac29e2a629775dcf54812a5373c88a23bbb5a6a951db2 +size 1979107 diff --git a/LanPaint/examples/Example_13/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_13/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..bfd20e873b4b091c9f5f0f9f7e619bf9e5e37e13 --- /dev/null +++ b/LanPaint/examples/Example_13/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e780881a5bcee55014f2269d4a14a5d8e47eaaa4050c63aa5316d9ca41a95ad6 +size 2262963 diff --git a/LanPaint/examples/Example_13/Original_No_Mask.png b/LanPaint/examples/Example_13/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..8b287cfeef7c4ace62e1e9b727c2b10863527bc9 --- /dev/null +++ b/LanPaint/examples/Example_13/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36645d07d92ea0a9ec14e5bb6d9f43494ef52850c00a94a3316f1ca4c6dfb339 +size 2253703 diff --git a/LanPaint/examples/Example_14/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_14/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..b425c3a24991a03aa434e58c540e1f432a0f3113 --- /dev/null +++ b/LanPaint/examples/Example_14/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25297b70e1d4d52f3d55a5c2d641a7c7c0e51c57f9d30d02e323d21f6b8b383b +size 1400795 diff --git a/LanPaint/examples/Example_14/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_14/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..2dc40321763ae9c33919bfb0b517f04ca0f2faf6 --- /dev/null +++ b/LanPaint/examples/Example_14/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc5bd8fedd04343860d1b596b9c192c33c0c4c2d9c6638effe9d4df8f4ab7a9a +size 2634040 diff --git a/LanPaint/examples/Example_14/Original_No_Mask.png b/LanPaint/examples/Example_14/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..8210a209d8818bd9692b6b66f5be2f9394a44896 --- /dev/null +++ b/LanPaint/examples/Example_14/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0cc49737b1494e8fce8ab0e180b98de73d9e751821e76baa7a37cff639f3e3e +size 2301687 diff --git a/LanPaint/examples/Example_14/QwenEdit_2509_InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_14/QwenEdit_2509_InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..5ef13172325b993ce7a4afb599ba2797a3921c4e --- /dev/null +++ b/LanPaint/examples/Example_14/QwenEdit_2509_InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a273186bfa03b7ec2b53415c4a951f4a974fd94e690963ccf238aeabd706fb8 +size 1423596 diff --git a/LanPaint/examples/Example_15/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_15/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..c9036dd1fc8897c6c58a3190b6f88bc6c40452d3 --- /dev/null +++ b/LanPaint/examples/Example_15/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1163fcaf154ebf138d0e7998854d664dbc0e0c40bac3d4164ab911f078b967f0 +size 1461257 diff --git a/LanPaint/examples/Example_15/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_15/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..a48373e929b399bce17f78e4dba42d10518c3579 --- /dev/null +++ b/LanPaint/examples/Example_15/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0122fc252e7c7e215efc47b26d0ab3f92fb02a19f2fbbec9d56559767a327ca1 +size 1687942 diff --git a/LanPaint/examples/Example_15/Original_No_Mask.png b/LanPaint/examples/Example_15/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..79cb245a12edeb606b131f8c7f8e81c1bf0aabee --- /dev/null +++ b/LanPaint/examples/Example_15/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0283dab9de5f121152041a8e171f0b79bac92c2cd092d3f16282db8990bc97 +size 1685017 diff --git a/LanPaint/examples/Example_16/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_16/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..7522562af6b417d929218307ec3260e9e90c152e --- /dev/null +++ b/LanPaint/examples/Example_16/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7eab9dcd421759b72f2bfd71c4da2d0bacded7c91f249677f0f86fe7ca2c69b +size 1463567 diff --git a/LanPaint/examples/Example_16/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_16/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..a48373e929b399bce17f78e4dba42d10518c3579 --- /dev/null +++ b/LanPaint/examples/Example_16/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0122fc252e7c7e215efc47b26d0ab3f92fb02a19f2fbbec9d56559767a327ca1 +size 1687942 diff --git a/LanPaint/examples/Example_16/Original_No_Mask.png b/LanPaint/examples/Example_16/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..79cb245a12edeb606b131f8c7f8e81c1bf0aabee --- /dev/null +++ b/LanPaint/examples/Example_16/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0283dab9de5f121152041a8e171f0b79bac92c2cd092d3f16282db8990bc97 +size 1685017 diff --git a/LanPaint/examples/Example_17/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 b/LanPaint/examples/Example_17/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f987ba3c483057c323b999704b6f6f223226dc0d --- /dev/null +++ b/LanPaint/examples/Example_17/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:385830dbfbd8e787fcf4d49374db8fd95954ff85b3847a17990b34cc01763c50 +size 242013 diff --git a/LanPaint/examples/Example_17/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_17/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..654e38ae2e0ec82b0fda25ed8046af369496c8bb --- /dev/null +++ b/LanPaint/examples/Example_17/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a05b874d181c2420484a3ef52d1bd7b51b26b575cce2430b56a2930ddd61d4c9 +size 226265 diff --git a/LanPaint/examples/Example_17/Original_No_Mask.mp4 b/LanPaint/examples/Example_17/Original_No_Mask.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0ba39d57e9b361cac18810fc58303234c9fc570 --- /dev/null +++ b/LanPaint/examples/Example_17/Original_No_Mask.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:068b1833025a50f0d1d3d4331900cf3adaa4dcc6ac42c6f88aec52c9134aa565 +size 141618 diff --git a/LanPaint/examples/Example_17/Wan22_ 5B_Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 b/LanPaint/examples/Example_17/Wan22_ 5B_Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..537c83db16448497e31f161863aa38cd8ed57f42 --- /dev/null +++ b/LanPaint/examples/Example_17/Wan22_ 5B_Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f80daf4c7280a62cd6a6379089d12568a31ba15b92e793ecba6336708c50967 +size 182341 diff --git a/LanPaint/examples/Example_18/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 b/LanPaint/examples/Example_18/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb3c2f956c4939a7da87ccdadabd4169ffebc074 --- /dev/null +++ b/LanPaint/examples/Example_18/Inpainted_40frames_Drag_Me_to_ComfyUI.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4665bc511bb32e83577d89742d62ea61b13666b566f0d952b49b848138c2f6e +size 191246 diff --git a/LanPaint/examples/Example_18/Inpainted_81frames_Drag_Me_to_ComfyUI.mp4 b/LanPaint/examples/Example_18/Inpainted_81frames_Drag_Me_to_ComfyUI.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3f12f60839917ad7cf9e836e5019091980710a7a --- /dev/null +++ b/LanPaint/examples/Example_18/Inpainted_81frames_Drag_Me_to_ComfyUI.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:260eeaf1481a58865b72a69c7e2853683e978cb67d4a3a4fe81e0eaa0ad3b952 +size 254449 diff --git a/LanPaint/examples/Example_18/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_18/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..281ffdb00904d0feb297ed8664e5b142faa5b8ee --- /dev/null +++ b/LanPaint/examples/Example_18/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78d21c86e90da40022692afc481d9642886b069d9c03833b392f52c905fb43f2 +size 144611 diff --git a/LanPaint/examples/Example_18/Original_No_Mask.mp4 b/LanPaint/examples/Example_18/Original_No_Mask.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..080ea59f2f599dfe380519f4eb63670ecd14047e --- /dev/null +++ b/LanPaint/examples/Example_18/Original_No_Mask.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0d3a6911b7153e5717afefeb4223cd177439f807a7098b860bdb3efcba963fb +size 167656 diff --git a/LanPaint/examples/Example_19/Original_Load_Me_in_Loader.mp4 b/LanPaint/examples/Example_19/Original_Load_Me_in_Loader.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f6bba31ef50652d887393b6d6b49fe09de9473b --- /dev/null +++ b/LanPaint/examples/Example_19/Original_Load_Me_in_Loader.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f824aaddd47c37cd251ffaa4ba112a50a1f24d4b0b1af9ab96221d8d5d5824b2 +size 355764 diff --git a/LanPaint/examples/Example_19/Outpainted_40frames_Drag_Me_to_ComfyUI.mp4 b/LanPaint/examples/Example_19/Outpainted_40frames_Drag_Me_to_ComfyUI.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e4d38187a65bc0ba80d37b995dfd671d92fe9a3 --- /dev/null +++ b/LanPaint/examples/Example_19/Outpainted_40frames_Drag_Me_to_ComfyUI.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bcca42fcfed7ae8f320b5f00a6a07e03d1e7b9b16e6c9f6a88659fc37f85c2a +size 690642 diff --git a/LanPaint/examples/Example_19/Outpainted_41frames_Drag_Me_to_ComfyUI.mp4 b/LanPaint/examples/Example_19/Outpainted_41frames_Drag_Me_to_ComfyUI.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..900f8b6d82a46fa175daeef31cdb6fb59b8f91a3 --- /dev/null +++ b/LanPaint/examples/Example_19/Outpainted_41frames_Drag_Me_to_ComfyUI.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b37244524277a125f87f00a2d1b2a80697fcdfbb54d4b80e2fc85777049706fb +size 757591 diff --git a/LanPaint/examples/Example_2/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_2/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..bc8f7eaf70be0d21487afd9336bcff7edd4b2e87 --- /dev/null +++ b/LanPaint/examples/Example_2/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb478ff2bdc9a41cd2914ae7068d94bce509e671839b33655728c2b53825b769 +size 1497073 diff --git a/LanPaint/examples/Example_2/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_2/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..6a8f7632296762c12ae833487f7e9b9a828670bb --- /dev/null +++ b/LanPaint/examples/Example_2/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e95dc65b370557b47053a07881736edb1332c87e2bb737a9feb5ad4e534c457b +size 1680231 diff --git a/LanPaint/examples/Example_2/Original_No_Mask.png b/LanPaint/examples/Example_2/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..6127c0cd0273f3b1676e47b1e6e28825109eaff3 --- /dev/null +++ b/LanPaint/examples/Example_2/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:828b21fbd0c062c73a9f22a7d89555342eb7515ae8347ca8cc0aadb15fe74f8d +size 1481450 diff --git a/LanPaint/examples/Example_20/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_20/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..199c4c2e1e02b6de68087dee77e6252b2b49d689 --- /dev/null +++ b/LanPaint/examples/Example_20/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:947a3e8d08c639f1e32d8d041e6b57d7f2c4bd8f146630a2db240763dee7cbe3 +size 1404567 diff --git a/LanPaint/examples/Example_20/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_20/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..a48373e929b399bce17f78e4dba42d10518c3579 --- /dev/null +++ b/LanPaint/examples/Example_20/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0122fc252e7c7e215efc47b26d0ab3f92fb02a19f2fbbec9d56559767a327ca1 +size 1687942 diff --git a/LanPaint/examples/Example_20/Original_No_Mask.png b/LanPaint/examples/Example_20/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..79cb245a12edeb606b131f8c7f8e81c1bf0aabee --- /dev/null +++ b/LanPaint/examples/Example_20/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0283dab9de5f121152041a8e171f0b79bac92c2cd092d3f16282db8990bc97 +size 1685017 diff --git a/LanPaint/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..324d9946cb20cc2adfa11953ef942ca5afc22990 --- /dev/null +++ b/LanPaint/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ecb6db6463c48094e0379105f6765188131d2a3de29a133e39746b6253f548c +size 1201362 diff --git a/LanPaint/examples/Example_21/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_21/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..b82c582354d74bd21fce9d5de1b8e968667d9ee5 --- /dev/null +++ b/LanPaint/examples/Example_21/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:158c9ec43189f3b9c7b93ea4b26fc5da3444aa2520d9b7c0b8cd775ca3e80176 +size 1333516 diff --git a/LanPaint/examples/Example_21/Original_No_Mask.png b/LanPaint/examples/Example_21/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..4e2065d139d681891445787356eb1eca4bbbf9fa --- /dev/null +++ b/LanPaint/examples/Example_21/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baf2d6495f70848f05f40a8597ede5464bc967b8aadcfaed8f725ab1d3fe98e0 +size 1503853 diff --git a/LanPaint/examples/Example_22/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_22/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..ae50bc87cf55c99002f840882605ef7643276ad9 --- /dev/null +++ b/LanPaint/examples/Example_22/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abf8cbfae31632f533fe490b02682520e7e05e68ef6eac7846256ed409c559b0 +size 2030827 diff --git a/LanPaint/examples/Example_22/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_22/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..46ba6c6b079ae41746e967f1d7544720c24d9cef --- /dev/null +++ b/LanPaint/examples/Example_22/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c58e278be503882194244b551a97f97cbb04a5317f953a3d3fae6c5ad8fad3dd +size 1236694 diff --git a/LanPaint/examples/Example_22/Original_No_Mask.png b/LanPaint/examples/Example_22/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..4e2065d139d681891445787356eb1eca4bbbf9fa --- /dev/null +++ b/LanPaint/examples/Example_22/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baf2d6495f70848f05f40a8597ede5464bc967b8aadcfaed8f725ab1d3fe98e0 +size 1503853 diff --git a/LanPaint/examples/Example_23/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_23/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..164c8ed67032e434eb8d5867607b3e29138ab26e --- /dev/null +++ b/LanPaint/examples/Example_23/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c6abff8592790ad97e096538ea2c7e370c3584feb2331619a45b3bd3e96c399 +size 1690584 diff --git a/LanPaint/examples/Example_23/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_23/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..7dc2bf82f34c0369c26cf20d6eb129b9f89c18c9 --- /dev/null +++ b/LanPaint/examples/Example_23/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a9f63cf9e455e5d7bce6292ddd886700144ff44c8097936b72a9f0a81d4e386 +size 1896436 diff --git a/LanPaint/examples/Example_23/Original_No_Mask.png b/LanPaint/examples/Example_23/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..00fb4011e7a786c8a66ad478e8ccc20c676489b2 --- /dev/null +++ b/LanPaint/examples/Example_23/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdf1a2366ff91f8f6c8ff62a65b296c145d3a32b2036d5b81734d284c3da7897 +size 1654234 diff --git a/LanPaint/examples/Example_24/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_24/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..ab0c5d97fe2662bf4049b9f8630a56c2de89b02d --- /dev/null +++ b/LanPaint/examples/Example_24/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21d6ff4fe4e4e248da5c218e2b213cba59008165913719da81df54bab23b3ea2 +size 1855271 diff --git a/LanPaint/examples/Example_24/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_24/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0b128032b74f870d21c3922a88fa10786dc39b --- /dev/null +++ b/LanPaint/examples/Example_24/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761483259db1627ebc40ee8e94280e23bc374cc1afb02ca2f344416fcd1ab50d +size 1607208 diff --git a/LanPaint/examples/Example_24/Original_No_Mask.png b/LanPaint/examples/Example_24/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..d47e1d2e9a0d52fd1acedbd4b5c67a77fd134729 --- /dev/null +++ b/LanPaint/examples/Example_24/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f647ea90aeefb71d4cfaa4081dacd824e8fc71a5421fb23d0af3cd81f2d0cce0 +size 1734637 diff --git a/LanPaint/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..4b314d54de3aeddec1c03cd0a34ed28a3a603348 --- /dev/null +++ b/LanPaint/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d895c7b550408ad76176f926391164e28a2afdc98f6c295461434d7e815497ab +size 2284269 diff --git a/LanPaint/examples/Example_25/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_25/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..3da07aed70a417947ac791efd71c32bdf97b5ce3 --- /dev/null +++ b/LanPaint/examples/Example_25/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:222e9e62883758983db632b2d3ae405cf59f3fb9e5a0a2d27f3ba6c6d9b3b293 +size 1894508 diff --git a/LanPaint/examples/Example_25/Original_No_Mask.png b/LanPaint/examples/Example_25/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..ff30011825650bd5db2426364a4f800f797df1db --- /dev/null +++ b/LanPaint/examples/Example_25/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f522cc0fd31b3b3c601651ac63e2c81b0454acf85619918f1e6f3bf929e042bf +size 2891641 diff --git a/LanPaint/examples/Example_3/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_3/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..7ed7c308ddd007d963555dfcbe996adb354b04a6 --- /dev/null +++ b/LanPaint/examples/Example_3/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a66153a1988aeb9b1f880f35badee4f46a8f4ed35cc1f29d08855f9c8036801 +size 1300442 diff --git a/LanPaint/examples/Example_3/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_3/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..65ff51d93b46d6336824d8572fe9cf2d37139ebf --- /dev/null +++ b/LanPaint/examples/Example_3/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:038d3eb9f8592408b28c209af97927d7dc4ac543ddf1b3d7a560f81311dddea3 +size 1480882 diff --git a/LanPaint/examples/Example_3/Original_No_Mask.png b/LanPaint/examples/Example_3/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..937c18e81cab21a5bf2be568dad22813731bf55b --- /dev/null +++ b/LanPaint/examples/Example_3/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46eb04f9c7387ce4befe7b0cdc42a2d7f2c332b68a2b4e6c382bd06fdc6350a9 +size 1292047 diff --git a/LanPaint/examples/Example_4/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_4/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..afaabbe9e61c60028fd82b11465ccb6c6a2afe45 --- /dev/null +++ b/LanPaint/examples/Example_4/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efb4fff6cfac048a97227fef52bfb34cdbbd53f1c74db25982f8c59bedc2f102 +size 1036191 diff --git a/LanPaint/examples/Example_4/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_4/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..a75be258255bb9ebdb8e12f9c6d11347b25d0e41 --- /dev/null +++ b/LanPaint/examples/Example_4/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33c42e51d999ae6442d2a4340a5fab4fd31e7ca430d92e8fe04fd633ef209897 +size 1185754 diff --git a/LanPaint/examples/Example_4/Original_No_Mask.png b/LanPaint/examples/Example_4/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..1a0c4b731e00c7f7723f019cf536db88193b710a --- /dev/null +++ b/LanPaint/examples/Example_4/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:950db549a240c9376a8e3f65a912777b6589206f936757d1573aca5fbb2c5f91 +size 1015973 diff --git a/LanPaint/examples/Example_5/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_5/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..2ca217ca73ebae8d465a07668c021c319006623b --- /dev/null +++ b/LanPaint/examples/Example_5/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:458af733e71700aba445835427e6b80a185a90c68eb095d506c882dbb52cffad +size 1079917 diff --git a/LanPaint/examples/Example_5/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_5/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..dc2219fc2082e56eb80fa952aa521afd98628f7d --- /dev/null +++ b/LanPaint/examples/Example_5/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33a992feb239126c73bd142a34d77f65db14d3b7ea42322248fc8ec0408700af +size 1188980 diff --git a/LanPaint/examples/Example_5/Original_No_Mask.png b/LanPaint/examples/Example_5/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..1a0c4b731e00c7f7723f019cf536db88193b710a --- /dev/null +++ b/LanPaint/examples/Example_5/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:950db549a240c9376a8e3f65a912777b6589206f936757d1573aca5fbb2c5f91 +size 1015973 diff --git a/LanPaint/examples/Example_6/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_6/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..5988476ba00884976c8dce07c63a32c49fee4c86 --- /dev/null +++ b/LanPaint/examples/Example_6/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61abc7eddea2d3f84c577f915ee0b8e197de7289800ff87e2a043aa62adc52a9 +size 1120740 diff --git a/LanPaint/examples/Example_6/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_6/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..8964ff901332b7e953619b2d369b3f032a660796 --- /dev/null +++ b/LanPaint/examples/Example_6/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fffdf28a6e513626bfe9b73481d2f47d7d8efabd8ef292a12d47d34eba4841e8 +size 876447 diff --git a/LanPaint/examples/Example_6/Original_No_Mask.png b/LanPaint/examples/Example_6/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..32dcd212d309a563dbeb7eee35b312ebce4af904 --- /dev/null +++ b/LanPaint/examples/Example_6/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09cc741530696d65c65e04bb37d6a9aae347a810e7bc7a975ad4df3e28fec610 +size 824464 diff --git a/LanPaint/examples/Example_7/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_7/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..ef3f853dc17f02baa4b3d1b0ef018b29ff633a94 --- /dev/null +++ b/LanPaint/examples/Example_7/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dd0733de4c62366cc26f3bab36c440a1ddb1c52b4cbb4708cfa1e5e86fda4e0 +size 1217100 diff --git a/LanPaint/examples/Example_7/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_7/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..c222c91023d8bf2c51de01c4ad7e9f8425b40c12 --- /dev/null +++ b/LanPaint/examples/Example_7/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f93f154770605f1e194ef62f82829b811e0d709971437dfe5786c62a1b8c79 +size 1427163 diff --git a/LanPaint/examples/Example_7/Original_No_Mask.png b/LanPaint/examples/Example_7/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..ef45fa17a59e796c5489bdc445ebfa7cc4dbf3e6 --- /dev/null +++ b/LanPaint/examples/Example_7/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2c5c79e557a45fde49bec20ca03efe8e94d171e3c0efda97f1ea46a6645716 +size 1401363 diff --git a/LanPaint/examples/Example_8/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_8/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..e1c6b3cc95886566787ff7d1f0e8114bb137fb8e --- /dev/null +++ b/LanPaint/examples/Example_8/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2019812155f293b55af748db3474fc97d6f199d449b1c4044afb28ecaebef246 +size 1459253 diff --git a/LanPaint/examples/Example_8/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_8/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..524f3a076559aeca92589daee4b9e2c2d5584dfa --- /dev/null +++ b/LanPaint/examples/Example_8/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:467139b9e026728a6e998bfb19cc0b42320765c23db954329a85092314de2292 +size 1616700 diff --git a/LanPaint/examples/Example_8/Original_No_Mask.png b/LanPaint/examples/Example_8/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..239ca9462026a494c8d087b4eb42c87592239d4f --- /dev/null +++ b/LanPaint/examples/Example_8/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1066c4b73b56adb997381af064e11c804f9474d01a51db4375f6662bae7352dc +size 1389123 diff --git a/LanPaint/examples/Example_9/InPainted_Drag_Me_to_ComfyUI.png b/LanPaint/examples/Example_9/InPainted_Drag_Me_to_ComfyUI.png new file mode 100644 index 0000000000000000000000000000000000000000..88be726ec7139160893c485cbdd6e4b9bdc2fa85 --- /dev/null +++ b/LanPaint/examples/Example_9/InPainted_Drag_Me_to_ComfyUI.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75730c8781dd8acfe8b0281df4a0d20d4ca1b58dfc51b309051018aac793925f +size 1421249 diff --git a/LanPaint/examples/Example_9/Masked_Load_Me_in_Loader.png b/LanPaint/examples/Example_9/Masked_Load_Me_in_Loader.png new file mode 100644 index 0000000000000000000000000000000000000000..5c0929106a8c2a127533a5cf00194e775edc956d --- /dev/null +++ b/LanPaint/examples/Example_9/Masked_Load_Me_in_Loader.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2920e791b922d8bbc901828731c0d72b4f1383d7a077b0c936f08525a7a0aad3 +size 1619485 diff --git a/LanPaint/examples/Example_9/Original_No_Mask.png b/LanPaint/examples/Example_9/Original_No_Mask.png new file mode 100644 index 0000000000000000000000000000000000000000..d0ba803fd8d8191486377b9490f1023a7d04a9c3 --- /dev/null +++ b/LanPaint/examples/Example_9/Original_No_Mask.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:049bb0acff6af3ae1a7f43acf5309a15fbffdc352d7afd8b0f03443fe9254891 +size 1379780 diff --git a/LanPaint/examples/InpaintChara_04.jpg b/LanPaint/examples/InpaintChara_04.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e101592004d7df88278f9d72836437533da622ff --- /dev/null +++ b/LanPaint/examples/InpaintChara_04.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a196396cab7b98160dd007504b8036b70c8432323c93216747c50dd4e802a8bd +size 719312 diff --git a/LanPaint/examples/InpaintChara_05.jpg b/LanPaint/examples/InpaintChara_05.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b33b0b0b93b931d45c1ad291616fd9dd4d512c30 --- /dev/null +++ b/LanPaint/examples/InpaintChara_05.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f23ac797e3e60c56ac43d32001fd34b47360b7680aa826ab35c67b2b4ae0fbef +size 659507 diff --git a/LanPaint/examples/InpaintChara_06.jpg b/LanPaint/examples/InpaintChara_06.jpg new file mode 100644 index 0000000000000000000000000000000000000000..50c1a071d3fdd8938407c8c6bf3ef66cd563bd31 --- /dev/null +++ b/LanPaint/examples/InpaintChara_06.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:921db97a029150a8f075bbd1e396db11c9ec5e27daefa7d70b652e415445876e +size 365684 diff --git a/LanPaint/examples/InpaintChara_07.jpg b/LanPaint/examples/InpaintChara_07.jpg new file mode 100644 index 0000000000000000000000000000000000000000..308785f544b9572c5dae1ed9e78ae75cc52bcf07 --- /dev/null +++ b/LanPaint/examples/InpaintChara_07.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f139ace3aa5f120fe0e7264df9313fa992837c0c4b8ef82e66c4aa15e828e6c7 +size 357061 diff --git a/LanPaint/examples/InpaintChara_08.jpg b/LanPaint/examples/InpaintChara_08.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6bfa125688005dcb60a0e16e2513d2a87f0a15fd --- /dev/null +++ b/LanPaint/examples/InpaintChara_08.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58e2947531a63a9cbbccd5f088e14625da5f3af311265ec4af42ea78b8bdb0e6 +size 388017 diff --git a/LanPaint/examples/InpaintChara_09.jpg b/LanPaint/examples/InpaintChara_09.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7df88c5bd010403f16963d2d217b4103936f86a3 --- /dev/null +++ b/LanPaint/examples/InpaintChara_09.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47e04ff8f5fd566c6b5eb4e3fe366fa3cd9c6773031b00b4fb75f740be81ee0d +size 406887 diff --git a/LanPaint/examples/InpaintChara_10.jpg b/LanPaint/examples/InpaintChara_10.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fdf5a481394470ede00e838698045cc627d4318b --- /dev/null +++ b/LanPaint/examples/InpaintChara_10.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34035b0b3104ac0e880897d4f5e69ebaf25e06554affdf5d59ad78766634e246 +size 562381 diff --git a/LanPaint/examples/InpaintChara_11.jpg b/LanPaint/examples/InpaintChara_11.jpg new file mode 100644 index 0000000000000000000000000000000000000000..12cebdf8ebc4d194d3cac8b587630d02f66267e0 --- /dev/null +++ b/LanPaint/examples/InpaintChara_11.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58c66c4cb6076fbd519d00a87e4531e649e22c8feeae733bde630bba1e020f60 +size 620859 diff --git a/LanPaint/examples/InpaintChara_12.jpg b/LanPaint/examples/InpaintChara_12.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51c93c4f15fa8ff9b39366437f4e31478dca6fa5 --- /dev/null +++ b/LanPaint/examples/InpaintChara_12.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7233fcdef65ca503f15df459a58ff6e8212aeb8042d20f6e1f044aeb07eff8e3 +size 477936 diff --git a/LanPaint/examples/InpaintChara_13(1).jpg b/LanPaint/examples/InpaintChara_13(1).jpg new file mode 100644 index 0000000000000000000000000000000000000000..e56b7bad44dc9dd919d1cfad06159a6e7bdc65df --- /dev/null +++ b/LanPaint/examples/InpaintChara_13(1).jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dddc9fbddc3134a2dd3888fa22f3b1214427bf2b64dd1814ad728bd9a8cb4c61 +size 416620 diff --git a/LanPaint/examples/InpaintChara_13.jpg b/LanPaint/examples/InpaintChara_13.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a392230d873331a9432f26c1a7c26a65057c0189 --- /dev/null +++ b/LanPaint/examples/InpaintChara_13.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4841156fbd83551f79fbedcd0f2746da6e245fdae501f6d5069c35977605ae +size 800349 diff --git a/LanPaint/examples/InpaintChara_14.jpg b/LanPaint/examples/InpaintChara_14.jpg new file mode 100644 index 0000000000000000000000000000000000000000..52df8412a8d84feb648839ca65612fd7ab7ffc2c --- /dev/null +++ b/LanPaint/examples/InpaintChara_14.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8584ecb9f19c65708786b07e0e51feaab5cffc6738c8af50195b8cc74348a7 +size 266479 diff --git a/LanPaint/examples/InpaintChara_45.jpg b/LanPaint/examples/InpaintChara_45.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f7f1f636ed51a47cf448f156dce661fe92dec0a --- /dev/null +++ b/LanPaint/examples/InpaintChara_45.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16a630528c6329ef1ce221f4b7a2bcd13df6e41612ae2b994c5213293e696098 +size 308069 diff --git a/LanPaint/examples/InpaintChara_46.jpg b/LanPaint/examples/InpaintChara_46.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a5de08261db1c01d5da5810d127acc442dd2fbed --- /dev/null +++ b/LanPaint/examples/InpaintChara_46.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:472b19993bf69f43c2b98b734f4c155515ddf5a2fe0380ca4d24ec4b98bc1d89 +size 471719 diff --git a/LanPaint/examples/Inpainted_40frames_Drag_Me_to_ComfyUI_example17.gif b/LanPaint/examples/Inpainted_40frames_Drag_Me_to_ComfyUI_example17.gif new file mode 100644 index 0000000000000000000000000000000000000000..25caa2c0613d3e2894631dd92684712560c18f08 --- /dev/null +++ b/LanPaint/examples/Inpainted_40frames_Drag_Me_to_ComfyUI_example17.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df2eab583eb9c70abcd290895da5b54906d23a422a61c5136ec78ac0068c978a +size 4314608 diff --git a/LanPaint/examples/Inpainted_81frames_Drag_Me_to_ComfyUI_example18.gif b/LanPaint/examples/Inpainted_81frames_Drag_Me_to_ComfyUI_example18.gif new file mode 100644 index 0000000000000000000000000000000000000000..31aa51ebf2afae64c626c06d91e18a91ec5c1146 --- /dev/null +++ b/LanPaint/examples/Inpainted_81frames_Drag_Me_to_ComfyUI_example18.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec3c1bf21114a55afbb682ce449796ffb7dd7b73c17d5fa1ad572b38eefd11b6 +size 6125588 diff --git a/LanPaint/examples/LanPaintQwen_01.jpg b/LanPaint/examples/LanPaintQwen_01.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cfd594d5e5ed063cfa4e7fe36dad5bdfb284108e --- /dev/null +++ b/LanPaint/examples/LanPaintQwen_01.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:467e970267d7cb64a649f063ac566dfd9642f8400ba337f5a9d331e8a4026254 +size 298489 diff --git a/LanPaint/examples/LanPaintQwen_03.jpg b/LanPaint/examples/LanPaintQwen_03.jpg new file mode 100644 index 0000000000000000000000000000000000000000..28d0a278dd62e7deeefd50e50a4cfbd432d9e6b5 --- /dev/null +++ b/LanPaint/examples/LanPaintQwen_03.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c23295f65a0db516994a581aa26a72c02eb2891a8f28206aa44de35dd0fa308e +size 408341 diff --git a/LanPaint/examples/LanPaintQwen_04.jpg b/LanPaint/examples/LanPaintQwen_04.jpg new file mode 100644 index 0000000000000000000000000000000000000000..47895404bfc54e880cc1f1e1ee3c93b7832e6eac --- /dev/null +++ b/LanPaint/examples/LanPaintQwen_04.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:611a9b284179f527f7b663d79dd6c836a7d26820994bf1bf1af89c7d749d8f49 +size 819855 diff --git a/LanPaint/examples/Mask_Example19_.png b/LanPaint/examples/Mask_Example19_.png new file mode 100644 index 0000000000000000000000000000000000000000..0e01facda10e50b13353aec870ac4037e92e2413 --- /dev/null +++ b/LanPaint/examples/Mask_Example19_.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d22aa153e6659f1c03344082676c700691434c5bc1579e94d04af37d852ea6e +size 564437 diff --git a/LanPaint/examples/Original_Load_Me_in_Loader_example19.gif b/LanPaint/examples/Original_Load_Me_in_Loader_example19.gif new file mode 100644 index 0000000000000000000000000000000000000000..cdf26084bf9c6ff1f63ef1f0c49bf8c2195f5e99 --- /dev/null +++ b/LanPaint/examples/Original_Load_Me_in_Loader_example19.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cbda1d55c44d879d953acd4e977bfb51634a520d614cea2feacbe9e733b1d36 +size 6477602 diff --git a/LanPaint/examples/Original_No_Mask-example18.gif b/LanPaint/examples/Original_No_Mask-example18.gif new file mode 100644 index 0000000000000000000000000000000000000000..06a5c3ea7706928e724cbfd85164f432024ad00e --- /dev/null +++ b/LanPaint/examples/Original_No_Mask-example18.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17a5ae8206a1dc8287e25388efe3e8577fdfb7452ccd557f66482a99937281a7 +size 5069816 diff --git a/LanPaint/examples/Original_No_Mask_example17.gif b/LanPaint/examples/Original_No_Mask_example17.gif new file mode 100644 index 0000000000000000000000000000000000000000..3ae019b08647d63d9f001f27caccb8875208915b --- /dev/null +++ b/LanPaint/examples/Original_No_Mask_example17.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:812da8715229ea8039325aeabf5fa2f90067bcb46508d7b057bf6d17e528ab3b +size 4328113 diff --git a/LanPaint/examples/Outpainted_40frames_Drag_Me_to_ComfyUI_example19.gif b/LanPaint/examples/Outpainted_40frames_Drag_Me_to_ComfyUI_example19.gif new file mode 100644 index 0000000000000000000000000000000000000000..9ccb0ee3da53f44b770b255a2c10b3c82ef57b37 --- /dev/null +++ b/LanPaint/examples/Outpainted_40frames_Drag_Me_to_ComfyUI_example19.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d59eef32aa4719bd7945d658b91e916a2a7daf698330d8aa23c0f4010ca8b79 +size 8915993 diff --git a/LanPaint/pyproject.toml b/LanPaint/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..071d3ba41869059fc34c49daa83569f463575041 --- /dev/null +++ b/LanPaint/pyproject.toml @@ -0,0 +1,82 @@ +[build-system] +requires = ["setuptools>=70.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "LanPaint" +version = "1.5.0" +description = "Achieve seamless inpainting results without needing a specialized inpainting model." +authors = [ + {name = "LanPaint", email = "czhengac@connect.ust.hk"} +] +readme = "README.md" +license = {text = "GNU General Public License v3"} +classifiers = [] +dependencies = [ + +] + +[project.optional-dependencies] +dev = [ + "bump-my-version", + "coverage", # testing + "mypy", # linting + "pre-commit", # runs linting on commit + "pytest", # testing + "ruff", # linting +] + +[project.urls] +bugs = "https://github.com/scraed/LanPaint/issues" +homepage = "https://github.com/scraed/LanPaint" +Repository = "https://github.com/scraed/LanPaint" + +[tool.comfy] +PublisherId = "scraed" +DisplayName = "LanPaint" +Icon = "" + +[tool.setuptools.package-data] +"*" = ["*.*"] + +[tool.pytest.ini_options] +minversion = "8.0" +testpaths = [ + "tests", +] + +[tool.mypy] +files = "." + +# Use strict defaults +strict = true +warn_unreachable = true +warn_no_return = true + +[[tool.mypy.overrides]] +# Don't require test functions to include types +module = "tests.*" +allow_untyped_defs = true +disable_error_code = "attr-defined" + +[tool.ruff] +# extend-exclude = ["static", "ci/templates"] +line-length = 140 +src = ["src", "tests"] +target-version = "py39" + +# Add rules to ban exec/eval +[tool.ruff.lint] +select = [ + "S102", # exec-builtin + "S307", # eval-used + "W293", + "F", # The "F" series in Ruff stands for "Pyflakes" rules, which catch various Python syntax errors and undefined names. + # See all rules here: https://docs.astral.sh/ruff/rules/#pyflakes-f +] + +[tool.ruff.lint.per-file-ignores] +"src/LanPaint/nodes.py" = ["F403", "F405"] + +[tool.ruff.lint.flake8-quotes] +inline-quotes = "double" diff --git a/LanPaint/src/LanPaint.egg-info/PKG-INFO b/LanPaint/src/LanPaint.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..ab9cf677c06d7209168c7ad2979211d030385103 --- /dev/null +++ b/LanPaint/src/LanPaint.egg-info/PKG-INFO @@ -0,0 +1,585 @@ +Metadata-Version: 2.4 +Name: LanPaint +Version: 1.5.0 +Summary: Achieve seamless inpainting results without needing a specialized inpainting model. +Author-email: LanPaint +License: GNU General Public License v3 +Project-URL: bugs, https://github.com/scraed/LanPaint/issues +Project-URL: homepage, https://github.com/scraed/LanPaint +Project-URL: Repository, https://github.com/scraed/LanPaint +Description-Content-Type: text/markdown +License-File: LICENSE +Provides-Extra: dev +Requires-Dist: bump-my-version; extra == "dev" +Requires-Dist: coverage; extra == "dev" +Requires-Dist: mypy; extra == "dev" +Requires-Dist: pre-commit; extra == "dev" +Requires-Dist: pytest; extra == "dev" +Requires-Dist: ruff; extra == "dev" +Dynamic: license-file + +
+ +# LanPaint: Universal Inpainting Sampler with "Think Mode" +[![TMLR PDF](https://img.shields.io/badge/TMLR-PDF-8A2BE2?logo=openreview&logoColor=white)](https://openreview.net/pdf?id=JPC8JyOUSW) +[![Python Benchmark](https://img.shields.io/badge/🐍-Python_Benchmark-3776AB?logo=python)](https://github.com/scraed/LanPaintBench) +[![ComfyUI Extension](https://img.shields.io/badge/ComfyUI-Extension-7B5DFF)](https://github.com/comfyanonymous/ComfyUI) +[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-yellow?logo=huggingface&logoColor=white)](https://huggingface.co/charrywhite/LanPaint) +[![Blog](https://img.shields.io/badge/📝-Blog-9cf)](https://scraed.github.io/scraedBlog/) +[![GitHub stars](https://img.shields.io/github/stars/scraed/LanPaint)](https://github.com/scraed/LanPaint/stargazers) +[![Discord](https://img.shields.io/badge/Discord-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/yN5wYDE6W4) +
+ + +Universally applicable inpainting ability for every model. LanPaint sampler lets the model "think" through multiple iterations before denoising, enabling you to invest more computation time for superior inpainting quality. + +This is the official implementation of ["LanPaint: Training-Free Diffusion Inpainting with Asymptotically Exact and Fast Conditional Sampling"](https://arxiv.org/abs/2502.03491), accepted by TMLR. + +The repository is for ComfyUI extension. + +Diffusers Support: [LanPaint-Diffusers](https://github.com/charrywhite/LanPaint-diffusers) by [@charrywhite](https://github.com/charrywhite/) + +Benchmark code for paper reproduce: [LanPaintBench](https://github.com/scraed/LanPaintBench). + +## Citation + +``` +@article{ +zheng2025lanpaint, +title={LanPaint: Training-Free Diffusion Inpainting with Asymptotically Exact and Fast Conditional Sampling}, +author={Candi Zheng and Yuan Lan and Yang Wang}, +journal={Transactions on Machine Learning Research}, +issn={2835-8856}, +year={2025}, +url={https://openreview.net/forum?id=JPC8JyOUSW}, +note={} +} +``` +**🎉 NEW 2026: Join our discord!** + +[Join our Discord](https://discord.gg/yN5wYDE6W4) to share experiences, discuss features, and explore future development. + +**🎬 NEW: LanPaint now supports inpainting and outpainting based on Z-Image!** + +`v1.5.0` fixes an important hidden bug that reduced performance and could blur images (especially with `z-image-base`) and also boosts overall LanPaint performance across other models. + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Original_No_Mask.png) | ![Masked Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png) | + +**🎬 NEW: LanPaint now supports Z-Image-Base too!** + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Original_No_Mask.png) | ![Masked Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png) | + + +**🎬 NEW: LanPaint now supports video inpainting and outpainting based on Wan 2.2!** + +
+ +| Original Video | Mask (edit T-shirt text) | Inpainted Result | +|:--------------:|:----:|:----------------:| +| ![Original](https://github.com/scraed/LanPaint/blob/master/examples/Original_No_Mask-example18.gif) | ![Mask](https://github.com/scraed/LanPaint/blob/master/examples/Example_18/Masked_Load_Me_in_Loader.png) | ![Result](https://github.com/scraed/LanPaint/blob/master/examples/Inpainted_81frames_Drag_Me_to_ComfyUI_example18.gif) | + +*Video Inpainting Example: 81 frames with temporal consistency* + +
+ +Check our latest [Wan 2.2 Video Examples](#video-examples-beta), [Wan 2.2 Image Examples](#example-wan22-inpaintlanpaint-k-sampler-5-steps-of-thinking), and +[Qwen Image Edit 2509](#example-qwen-edit-2509-inpaint) support. + + +## Table of Contents +- [Features](#features) +- [Quickstart](#quickstart) +- [How to Use Examples](#how-to-use-examples) +- [Video Examples (Beta)](#video-examples-beta) + - [Wan 2.2 Video Inpainting](#wan-22-video-inpainting) + - [Wan 2.2 5B Video Inpainting](#wan-22-5b-video-inpainting) + - [Wan 2.2 Video Outpainting](#wan-22-video-outpainting) + - [Resource Consumption](#resource-consumption) +- [Image Examples](#image-examples) + - [Flux.2.Dev](#example-flux2dev-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Flux 2 klein](#example-flux-2-klein-inpaintlanpaint-k-sampler-2-steps-of-thinking) + - [Z-image](#example-z-image-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Z-image-base](#example-z-image-base-inpaintlanpaint-k-sampler-3-steps-of-thinking) + - [Hunyuan T2I](#example-hunyuan-t2i-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Wan 2.2 T2I](#example-wan22-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Wan 2.2 T2I with reference](#example-wan22-partial-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Qwen Image Edit 2511 2509](#example-qwen-edit-2509-inpaint) + - [Qwen Image Edit 2508](#example-qwen-edit-2508-inpaint) + - [Qwen Image](#example-qwen-image-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [HiDream](#example-hidream-inpaint-lanpaint-k-sampler-5-steps-of-thinking) + - [SD 3.5](#example-sd-35-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [Flux](#example-flux-inpaintlanpaint-k-sampler-5-steps-of-thinking) + - [SDXL](#example-sdxl-0-character-consistency-side-view-generation-lanpaint-k-sampler-5-steps-of-thinking) +- [Usage](#usage) + - [Basic Sampler](#basic-sampler) + - [Advanced Sampler](#lanpaint-ksampler-advanced) + - [Tuning Guide](#lanpaint-ksampler-advanced-tuning-guide) +- [Community Showcase](#community-showcase-) +- [FAQ](#faq) +- [Updates](#updates) +- [ToDo](#todo) +- [Citation](#citation) + +## Features + +- **Universal Compatibility** – Works instantly with almost any model (**Z-image, Z-image-base, Hunyuan, Wan 2.2, Qwen Image/Edit, HiDream, SD 3.5, Flux-series, SDXL, SD 1.5 or custom LoRAs**) and ControlNet. +![Inpainting Result 13](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_13.jpg) +- **No Training Needed** – Works out of the box with your existing model. +- **Easy to Use** – Same workflow as standard ComfyUI KSampler. +- **Flexible Masking** – Supports any mask shape, size, or position for inpainting/outpainting. +- **No Workarounds** – Generates 100% new content (no blending or smoothing) without relying on partial denoising. +- **Beyond Inpainting** – You can even use it as a simple way to generate consistent characters. + +**Warning**: LanPaint has degraded performance on distillation models, such as Flux.dev, due to a similar [issue with LORA training](https://medium.com/@zhiwangshi28/why-flux-lora-so-hard-to-train-and-how-to-overcome-it-a0c70bc59eaf). Please use low flux guidance (1.0-2.0) to mitigate this [issue](https://github.com/scraed/LanPaint/issues/30). + +## Quickstart + +1. **Install ComfyUI**: Follow the official [ComfyUI installation guide](https://docs.comfy.org/get_started) to set up ComfyUI on your system. Or ensure your ComfyUI version > 0.3.11. +2. **Install ComfyUI-Manager**: Add the [ComfyUI-Manager](https://github.com/ltdrdata/ComfyUI-Manager) for easy extension management. +3. **Install LanPaint Nodes**: + - **Via ComfyUI-Manager**: Search for "[LanPaint](https://registry.comfy.org/publishers/scraed/nodes/LanPaint)" in the manager and install it directly. + - **Manually**: Click "Install via Git URL" in ComfyUI-Manager and input the GitHub repository link: + ``` + https://github.com/scraed/LanPaint.git + ``` + Alternatively, clone this repository into the `ComfyUI/custom_nodes` folder. +4. **Restart ComfyUI**: Restart ComfyUI to load the LanPaint nodes. + +Once installed, you'll find the LanPaint nodes under the "sampling" category in ComfyUI. Use them just like the default KSampler for high-quality inpainting! + + +## **How to Use Examples:** +1. Navigate to the **example** folder (i.e example_1), download all pictures. +2. Drag **InPainted_Drag_Me_to_ComfyUI.png** into ComfyUI to load the workflow. +3. Download the required model (i.e clicking **Model Used in This Example**). +4. Load the model in ComfyUI. +5. Upload **Masked_Load_Me_in_Loader.png** to the **"Load image"** node in the **"Mask image for inpainting"** group (second from left), or the **Prepare Image** node. +7. Queue the task, you will get inpainted results from LanPaint. Some example also gives you inpainted results from the following methods for comparison: + - **[VAE Encode for Inpainting](https://comfyanonymous.github.io/ComfyUI_examples/inpaint/)** + - **[Set Latent Noise Mask](https://comfyui-wiki.com/en/tutorial/basic/how-to-inpaint-an-image-in-comfyui)** + +## Video Examples (Beta) + +LanPaint now supports video inpainting with Wan 2.2, enabling you to seamlessly inpaint masked regions across video frames while maintaining temporal consistency. + +**Note:** LanPaint supports video inpainting for longer sequences (e.g., 81 frames), but processing time increases significantly (please check the [Resource Consumption](#resource-consumption) section for details) and performance may become unstable. For optimal results and stability, we recommend limiting video inpainting to **40 frames or fewer**. + +### Wan 2.2 Video Inpainting + +*Example: Wan2.2 t2v 14B, 480p video (11:6), 40 frames, LanPaint K Sampler, 2 steps of thinking* + +| Original Video | Mask (Add a white hat) | Inpainted Result | +|:--------------:|:----:|:----------------:| +| ![Original Video](https://github.com/scraed/LanPaint/blob/master/examples/Original_No_Mask_example17.gif) | ![Mask](https://github.com/scraed/LanPaint/blob/master/examples/Example_17/Masked_Load_Me_in_Loader.png) | ![Inpainted Result](https://github.com/scraed/LanPaint/blob/master/examples/Inpainted_40frames_Drag_Me_to_ComfyUI_example17.gif) | + +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_17) + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + +### Wan 2.2 5B Video Inpainting + +Similar to Wan 2.2 14B with slightly different workflow. [View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_17) + +### Wan 2.2 Video Outpainting + +Extend your videos beyond their original boundaries with LanPaint's video outpainting capability based on Wan 2.2. This feature allows you to expand the canvas of your videos while maintaining coherent motion and context. + +*Example: Wan2.2 t2v 14B, 480p video (1:1 outpaint to 11:6), 40 frames, LanPaint K Sampler, 2 steps of thinking* + +| Original Video | Mask (Expand to 880x480) | Outpainted Result | +|:--------------:|:----:|:-----------------:| +| ![Original Video](https://github.com/scraed/LanPaint/blob/master/examples/Original_Load_Me_in_Loader_example19.gif) | ![Mask](https://github.com/scraed/LanPaint/blob/master/examples/Mask_Example19_.png) | ![Outpainted Result](https://github.com/scraed/LanPaint/blob/master/examples/Outpainted_40frames_Drag_Me_to_ComfyUI_example19.gif) | + +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_19) + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + +### Resource Consumption + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Processing ModeResolutionFrames ProcessedVRAM RequiredTotal Runtime (20 steps)
Inpainting880×480 (11:6)40 frames39.8 GB05:37 min
Inpainting480×480 (1:1)40 frames38.0 GB05:35 min
Outpainting880×480 (11:6)40 frames40.2 GB05:36 min
Inpainting880×480 (11:6)81 frames43.3 GB16:23 min
Inpainting480×480 (1:1)81 frames39.8 GB14:25 min
Outpainting880×480 (11:6)81 frames42.6 GB13:46 min
+ +**Test Platform**: All tests were conducted on an NVIDIA RTX Pro 6000.
+**Model Used**: `wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors` and `wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors`.
+**Processing Steps**: 20 sampling steps x 2 (LanPaint steps of thinking).
+ +**Note:** Vram is required by the model, not LanPaint. To further reduce VRAM requirements, we recommend generating less frames and loading CLIP on CPU. + +## Image Examples + +### Example Hunyuan T2I: InPaint(LanPaint K Sampler, 5 steps of thinking) +We are excited to announce that LanPaint now supports inpainting with Hunyuan text to image generation. + +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_20) + + +You need to follow the ComfyUI version of [Hunyuan workflow](https://docs.comfy.org/tutorials/video/hunyuan-video#hunyuan-text-to-video-workflow) to download and install the model. + +### Example Wan2.2: InPaint(LanPaint K Sampler, 5 steps of thinking) +We are excited to announce that LanPaint now supports Wan2.2 text to image generation with Wan2.2 T2V model. + +![Inpainting Result 45](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_45.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_15) + + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + +### Example Z-image: InPaint(LanPaint K Sampler, 5 steps of thinking) +LanPaint also supports inpainting with the Z-image text-to-image model. + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Original_No_Mask.png) | ![Masked Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_21/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_21) + +
+View Z-image Outpainting (Original / Masked / Outpainted) + +| Original | Masked | Outpainted | +|:--------:|:------:|:----------:| +| ![Original Z-image Outpaint](https://github.com/scraed/LanPaint/blob/master/examples/Example_22/Original_No_Mask.png) | ![Masked Z-image Outpaint](https://github.com/scraed/LanPaint/blob/master/examples/Example_22/Masked_Load_Me_in_Loader.png) | ![Outpainted Z-image](https://github.com/scraed/LanPaint/blob/master/examples/Example_22/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Outpaint Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_22) + +You can download the Z-image model for ComfyUI from [Z-image](https://docs.comfy.org/zh-CN/tutorials/image/z-image/z-image-turbo). + +### Example Z-image-base: InPaint(LanPaint K Sampler, 3 steps of thinking) +LanPaint also supports inpainting with the Z-image-base model. + +**Warning (stability)**: Z-image-base can easily diverge with LanPaint. Start with **small `LanPaint_StepSize`** and **fewer thinking iterations** (lower `LanPaint_NumSteps`) and increase gradually only if stable. + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Original_No_Mask.png) | ![Masked Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/Masked_Load_Me_in_Loader.png) | ![Inpainted Z-image-base](https://github.com/scraed/LanPaint/blob/master/examples/Example_25/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_25) + +Workflow template (JSON): [Z_image_base_Inpaint.json](https://github.com/scraed/LanPaint/blob/master/example_workflows/Z_image_base_Inpaint.json) + +### Example Wan2.2: Partial InPaint(LanPaint K Sampler, 5 steps of thinking) +Sometimes we don't want to inpaint completely new content, but rather let the inpainted image reference the original image. One option to achieve this is to inpaint with an edit model like Qwen Image Edit. Another option is to perform a partial inpaint: allowing the diffusion process to start at some middle steps rather than from 0. + +![Inpainting Result 46](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_46.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_16) + + +You need to follow the ComfyUI version of [Wan2.2 T2V workflow](https://docs.comfy.org/tutorials/video/wan/wan2_2) to download and install the T2V model. + + +### Example Qwen Edit 2509: InPaint +Check our latest updated [Mased Qwen Edit Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_14) for Qwen Image Edit 2509. Download the model at [Qwen Image Edit 2509 Comfy](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models). This workflow also supports Qwen Image Edit 2511. + +![Qwen Result 3](https://github.com/scraed/LanPaint/blob/master/examples/LanPaintQwen_04.jpg) + +### Example Qwen Edit 2508: InPaint +![Qwen Result 2](https://github.com/scraed/LanPaint/blob/master/examples/LanPaintQwen_03.jpg) +Check [Mased Qwen Edit Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_14). You need to follow the ComfyUI version of [Qwen Image Edit workflow](https://docs.comfy.org/tutorials/image/qwen/qwen-image-edit) to download and install the model. + + + +### Example Qwen Image: InPaint(LanPaint K Sampler, 5 steps of thinking) + +![Inpainting Result 14](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_14.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_11) + + +You need to follow the ComfyUI version of [Qwen Image workflow](https://docs.comfy.org/tutorials/image/qwen/qwen-image) to download and install the model. + +The following examples utilize a random seed of 0 to generate a batch of 4 images for variance demonstration and fair comparison. (Note: Generating 4 images may exceed your GPU memory; please adjust the batch size as necessary.) + +![Qwen Result 1](https://github.com/scraed/LanPaint/blob/master/examples/LanPaintQwen_01.jpg) +Also check [Qwen Inpaint Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_13) and [Qwen Outpaint Workflow](https://github.com/scraed/LanPaint/tree/master/examples/Example_12). You need to follow the ComfyUI version of [Qwen Image workflow](https://docs.comfy.org/tutorials/image/qwen/qwen-image) to download and install the model. + +### Example HiDream: InPaint (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_11.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_8) + +You need to follow the ComfyUI version of [HiDream workflow](https://docs.comfy.org/tutorials/image/hidream/hidream-i1) to download and install the model. + +### Example HiDream: OutPaint(LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_13(1).jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_10) + +You need to follow the ComfyUI version of [HiDream workflow](https://docs.comfy.org/tutorials/image/hidream/hidream-i1) to download and install the model. Thanks [Amazon90](https://github.com/Amazon90) for providing this example. + +### Example SD 3.5: InPaint(LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_12.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_9) + +You need to follow the ComfyUI version of [SD 3.5 workflow](https://comfyui-wiki.com/en/tutorial/advanced/stable-diffusion-3-5-comfyui-workflow) to download and install the model. + +### Example Flux.2.Dev: InPaint(LanPaint K Sampler, 5 steps of thinking) + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Flux.2.Dev](https://github.com/scraed/LanPaint/blob/master/examples/Example_23/Original_No_Mask.png) | ![Masked Flux.2.Dev](https://github.com/scraed/LanPaint/blob/master/examples/Example_23/Masked_Load_Me_in_Loader.png) | ![Inpainted Flux.2.Dev](https://github.com/scraed/LanPaint/blob/master/examples/Example_23/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_23) + +[Model Used in This Example](https://huggingface.co/Comfy-Org/flux2-dev) + +(Note: Prompt First mode is disabled on Flux.2.Dev. As it does not use CFG guidance.) + +### Example Flux 2 klein: InPaint(LanPaint K Sampler, 2 steps of thinking) + +
+View Original / Masked / Inpainted Comparison + +| Original | Masked | Inpainted | +|:--------:|:------:|:---------:| +| ![Original Flux 2 klein](https://github.com/scraed/LanPaint/blob/master/examples/Example_24/Original_No_Mask.png) | ![Masked Flux 2 klein](https://github.com/scraed/LanPaint/blob/master/examples/Example_24/Masked_Load_Me_in_Loader.png) | ![Inpainted Flux 2 klein](https://github.com/scraed/LanPaint/blob/master/examples/Example_24/InPainted_Drag_Me_to_ComfyUI.png) | + +
+ +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_24) + +[Model Used in This Example](https://docs.comfy.org/zh-CN/tutorials/flux/flux-2-klein). If you have quality problem on Comfy 0.11 and 0.12, check [this issue](https://github.com/scraed/LanPaint/issues/80). + + +### Example Flux: InPaint(LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 7](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_10.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_7) +[Model Used in This Example](https://huggingface.co/Comfy-Org/flux1-dev/blob/main/flux1-dev-fp8.safetensors) +(Note: Prompt First mode is disabled on Flux. As it does not use CFG guidance.) + +### Example SDXL 0: Character Consistency (Side View Generation) (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 6](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_09.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_6) +[Model Used in This Example](https://civitai.com/models/1188071?modelVersionId=1408658) + +(Tricks 1: You can emphasize the character by copy it's image multiple times with Photoshop. Here I have made one extra copy.) + +(Tricks 2: Use prompts like multiple views, multiple angles, clone, turnaround. Use LanPaint's Prompt first mode (does not support Flux)) + +(Tricks 3: Remeber LanPaint can in-paint: Mask non-consistent regions and try again!) + + +### Example SDXL 1: Basket to Basket Ball (LanPaint K Sampler, 2 steps of thinking). +![Inpainting Result 1](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_04.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_1) +[Model Used in This Example](https://civitai.com/models/1188071?modelVersionId=1408658) +### Example SDXL 2: White Shirt to Blue Shirt (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 2](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_05.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_2) +[Model Used in This Example](https://civitai.com/models/1188071?modelVersionId=1408658) +### Example SDXL 3: Smile to Sad (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 3](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_06.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_3) +[Model Used in This Example](https://civitai.com/models/133005/juggernaut-xl) +### Example SDXL 4: Damage Restoration (LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 4](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_07.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_4) +[Model Used in This Example](https://civitai.com/models/133005/juggernaut-xl) +### Example SDXL 5: Huge Damage Restoration (LanPaint K Sampler, 20 steps of thinking) +![Inpainting Result 5](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_08.jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_5) +[Model Used in This Example](https://civitai.com/models/133005/juggernaut-xl) + +Check more for use cases like inpaint on [fine tuned models](https://github.com/scraed/LanPaint/issues/12#issuecomment-2938662021) and [face swapping](https://github.com/scraed/LanPaint/issues/12#issuecomment-2938723501), thanks to [Amazon90](https://github.com/Amazon90). + + +## Usage + +**Workflow Setup** +Same as default ComfyUI KSampler - simply replace with LanPaint KSampler nodes. The inpainting workflow is the same as the [SetLatentNoiseMask](https://comfyui-wiki.com/zh/comfyui-nodes/latent/inpaint/set-latent-noise-mask) inpainting workflow. + +**Note** +- LanPaint requires binary masks (values of 0 or 1) without opacity or smoothing. To ensure compatibility, set the mask's **opacity and hardness to maximum** in your mask editor. During inpainting, any mask with smoothing or gradients will automatically be converted to a binary mask. +- LanPaint relies heavily on your text prompts to guide inpainting - explicitly describe the content you want generated in the masked area. If results show artifacts or mismatched elements, counteract them with targeted negative prompts. + +## Basic Sampler +![Samplers](https://github.com/scraed/LanPaint/blob/master/Nodes.JPG) + +- LanPaint KSampler: The most basic and easy to use sampler for inpainting. +- LanPaint KSampler (Advanced): Full control of all parameters. + +### LanPaint KSampler +Simplified interface with recommended defaults: + +- Steps: 20 - 50. More steps will give more "thinking" and better results. +- LanPaint NumSteps: The turns of thinking before denoising. Recommend 5 for most of tasks ( which means 5 times slower than sampling without thinking). Use 10 for more challenging tasks. +- LanPaint Prompt mode: Image First mode and Prompt First mode. Image First mode focuses on the image, inpaint based on image context (maybe ignore prompt), while Prompt First mode focuses more on the prompt. Use Prompt First mode for tasks like character consistency. (Technically, it Prompt First mode change CFG scale to negative value in the BIG score to emphasis prompt, which will costs image quality.) + +### LanPaint KSampler (Advanced) +Full parameter control: +**Key Parameters** + +| Parameter | Range | Description | +|-----------|-------|-------------| +| `Steps` | 0-100 | Total steps of diffusion sampling. Higher means better inpainting. Recommend 20-50. | +| `LanPaint_NumSteps` | 0-20 | Reasoning iterations per denoising step ("thinking depth"). Easy task: 2-5. Hard task: 5-10 | +| `LanPaint_Lambda` | 0.1-50 | Content alignment strength (higher = stricter). Recommend 4.0 - 10.0 | +| `LanPaint_StepSize` | 0.1-1.0 | The StepSize of each thinking step. Recommend 0.1-0.5. | +| `LanPaint_Beta` | 0.1-2.0 | The StepSize ratio between masked / unmasked region. Small value can compensate high lambda values. Recommend 1.0 | +| `LanPaint_Friction` | 0.0-100.0 | The friction of Langevin dynamics. Higher means more slow but stable, lower means fast but unstable. Recommend 10.0 - 20.0| +| `LanPaint_EarlyStop` | 0-10 | Stop LanPaint iteration before the final sampling step. Helps to remove artifacts in some cases. Recommend 1-5| +| `LanPaint_PromptMode` | Image First / Prompt First | Image First mode focuses on the image context, maybe ignore prompt. Prompt First mode focuses more on the prompt. | + +For detailed descriptions of each parameter, simply hover your mouse over the corresponding input field to view tooltips with additional information. + +### LanPaint Mask Blend +This node blends the original image with the inpainted image based on the mask. It is useful if you want the unmasked region to match the original image pixel perfectly. + +## LanPaint KSampler (Advanced) Tuning Guide +For challenging inpainting tasks: + +1️⃣ **Boost Quality** +Increase **total number of sampling steps** (very important!), **LanPaint_NumSteps** (thinking iterations) or **LanPaint_Lambda** if the inpainted result does not meet your expectations. + +2️⃣ **Boost Speed** +Decrease **LanPaint_NumSteps** to accelerate generation! If you want better results but still need fewer steps, consider: + - **Increasing LanPaint_StepSize** to speed up the thinking process. + - **Decreasing LanPaint_Friction** to make the Langevin dynamics converges more faster. + +3️⃣ **Fix Unstability**: +If you find the results have wired texture, try +- Reduce **LanPaint_Friction** to make the Langevin dynamics more stable. +- Reduce **LanPaint_StepSize** to use smaller step size. +- Reduce **LanPaint_Beta** if you are using a high lambda value. + +⚠️ **Notes**: +- For effective tuning, **fix the seed** and adjust parameters incrementally while observing the results. This helps isolate the impact of each setting. Better to do it with a batche of images to avoid overfitting on a single image. + +## Community Showcase [](#community-showcase-) + +Discover how the community is using LanPaint! Here are some user-created tutorials: + +- [Ai绘画进阶148-三大王炸!庆祝高允贞出道6周年!T8即将直播?当AI绘画学会深度思考?!万能修复神器LanPaint,万物皆可修!-T8 Comfyui教程](https://www.youtube.com/watch?v=Z4DSTv3UPJo) +- [Ai绘画进阶151-真相了!T8竟是个AI?!LanPaint进阶(二),人物一致性,多视角实验性测试,新参数讲解,工作流分享-T8 Comfyui教程](https://www.youtube.com/watch?v=landiRhvF3k) +- [重绘和三视图角色一致性解决新方案!LanPaint节点尝试](https://www.youtube.com/watch?v=X0WbXdm6FA0) +- [ComfyUI: HiDream with Perturbation Upscale, LanPaint Inpainting (Workflow Tutorial)](https://www.youtube.com/watch?v=2-mGe4QVIIw&t=2785s) +- [ComfyUI必备LanPaint插件超详细使用教程](https://plugin.aix.ink/archives/lanpaint) + +Submit a PR to add your tutorial/video here, or open an [Issue](https://github.com/scraed/LanPaint/issues) with details! + +## FAQ +[Working togather with crop&stitch](https://github.com/scraed/LanPaint/issues/46) + +## Updates +- 2026/03/02 + - `v1.5.0`: Fixed a hidden bug that hurt performance and caused image blur (especially on `z-image-base`), and improved overall LanPaint performance on other models too. +- 2026/01/30 + - Add Z-image-base documentation and Example_25 workflow images. +- 2025/08/08 + - Add Qwen image support +- 2025/06/21 + - Update the algorithm with enhanced stability and outpaint performance. + - Add outpaint example + - Supports Sampler Custom (Thanks to [MINENEMA](https://github.com/MINENEMA)) +- 2025/06/04 + - Add more sampler support. + - Add early stopping to advanced sampler. +- 2025/05/28 + - Major update on the Langevin solver. It is now much faster and more stable. + - Greatly simplified the parameters for advanced sampler. + - Fix performance issue on Flux and SD 3.5 +- 2025/04/16 + - Added Primary HiDream support +- 2025/03/22 + - Added Primary Flux support + - Added Tease Mode +- 2025/03/10 + - LanPaint has received a major update! All examples now use the LanPaint K Sampler, offering a simplified interface with enhanced performance and stability. +- 2025/03/06: + - Bug Fix for str not callable error and unpack error. Big thanks to [jamesWalker55](https://github.com/jamesWalker55) and [EricBCoding](https://github.com/EricBCoding). + +## ToDo +- Try Implement Detailer +- ~~Provide inference code on without GUI.~~ Check our local Python benchmark code [LanPaintBench](https://github.com/scraed/LanPaintBench). + + +## Citation + +``` +@article{ +zheng2025lanpaint, +title={LanPaint: Training-Free Diffusion Inpainting with Asymptotically Exact and Fast Conditional Sampling}, +author={Candi Zheng and Yuan Lan and Yang Wang}, +journal={Transactions on Machine Learning Research}, +issn={2835-8856}, +year={2025}, +url={https://openreview.net/forum?id=JPC8JyOUSW}, +note={} +} +``` + + + + + diff --git a/LanPaint/src/LanPaint.egg-info/SOURCES.txt b/LanPaint/src/LanPaint.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e8e2e4493578f3f9a36c237868d38b826a945b0 --- /dev/null +++ b/LanPaint/src/LanPaint.egg-info/SOURCES.txt @@ -0,0 +1,19 @@ +LICENSE +MANIFEST.in +README.md +pyproject.toml +src/LanPaint/__init__.py +src/LanPaint/earlystop.py +src/LanPaint/lanpaint.py +src/LanPaint/nodes.py +src/LanPaint/types.py +src/LanPaint/utils.py +src/LanPaint.egg-info/PKG-INFO +src/LanPaint.egg-info/SOURCES.txt +src/LanPaint.egg-info/dependency_links.txt +src/LanPaint.egg-info/requires.txt +src/LanPaint.egg-info/top_level.txt +tests/test_LanPaint.py +tests/test_lanpaint_semantic_stop.py +tests/test_reshape_mask.py +tests/test_sho_regression.py \ No newline at end of file diff --git a/LanPaint/src/LanPaint.egg-info/dependency_links.txt b/LanPaint/src/LanPaint.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/LanPaint/src/LanPaint.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/LanPaint/src/LanPaint.egg-info/requires.txt b/LanPaint/src/LanPaint.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..e2cf126e59e1387d6c898f5d834cc9a4db891f08 --- /dev/null +++ b/LanPaint/src/LanPaint.egg-info/requires.txt @@ -0,0 +1,8 @@ + +[dev] +bump-my-version +coverage +mypy +pre-commit +pytest +ruff diff --git a/LanPaint/src/LanPaint.egg-info/top_level.txt b/LanPaint/src/LanPaint.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..0593c4b03a6b33715a94b32729cbb1a3abc2a774 --- /dev/null +++ b/LanPaint/src/LanPaint.egg-info/top_level.txt @@ -0,0 +1 @@ +LanPaint diff --git a/LanPaint/src/LanPaint/__init__.py b/LanPaint/src/LanPaint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LanPaint/src/LanPaint/__pycache__/__init__.cpython-312.pyc b/LanPaint/src/LanPaint/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cba301e8ac54be0d770f3de8ad1ec127101d64d4 Binary files /dev/null and b/LanPaint/src/LanPaint/__pycache__/__init__.cpython-312.pyc differ diff --git a/LanPaint/src/LanPaint/__pycache__/earlystop.cpython-312.pyc b/LanPaint/src/LanPaint/__pycache__/earlystop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33fbd4f8c2b85a52db873785f9b6b572243bcfbc Binary files /dev/null and b/LanPaint/src/LanPaint/__pycache__/earlystop.cpython-312.pyc differ diff --git a/LanPaint/src/LanPaint/__pycache__/lanpaint.cpython-312.pyc b/LanPaint/src/LanPaint/__pycache__/lanpaint.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2eadbacb75239e11bc202111ab7713e5a78b1630 Binary files /dev/null and b/LanPaint/src/LanPaint/__pycache__/lanpaint.cpython-312.pyc differ diff --git a/LanPaint/src/LanPaint/__pycache__/nodes.cpython-312.pyc b/LanPaint/src/LanPaint/__pycache__/nodes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89483d9b77625bf3f452f11083e5d0a447af6695 Binary files /dev/null and b/LanPaint/src/LanPaint/__pycache__/nodes.cpython-312.pyc differ diff --git a/LanPaint/src/LanPaint/__pycache__/types.cpython-312.pyc b/LanPaint/src/LanPaint/__pycache__/types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b2cb4bced1d27bbd30e8d240c48f475c9ec7a72 Binary files /dev/null and b/LanPaint/src/LanPaint/__pycache__/types.cpython-312.pyc differ diff --git a/LanPaint/src/LanPaint/__pycache__/utils.cpython-312.pyc b/LanPaint/src/LanPaint/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f7367a36d74ec9df33279658e9e2ec57c4f1286 Binary files /dev/null and b/LanPaint/src/LanPaint/__pycache__/utils.cpython-312.pyc differ diff --git a/LanPaint/src/LanPaint/earlystop.py b/LanPaint/src/LanPaint/earlystop.py new file mode 100644 index 0000000000000000000000000000000000000000..c1dc244d5b865d60686ce481467af6f471b2226c --- /dev/null +++ b/LanPaint/src/LanPaint/earlystop.py @@ -0,0 +1,337 @@ +""" +Early Stop Logic Contributed by `https://github.com/godnight10061`. +""" + +import inspect +from typing import Any, Callable, Optional + +import torch + +from .types import LangevinState + + +def _clamp01(val: float) -> float: + if val <= 0.0: + return 0.0 + if val >= 1.0: + return 1.0 + return val + + +def _abt_scale(abt_val: float) -> float: + """ + Smooth, parameter-free scale based on outer-step noise level. + + - 0 at abt=0/1 (disable at extreme noise / extreme tail) + - 1 at abt=0.5 (mid-schedule) + """ + abt_val = _clamp01(abt_val) + return _clamp01(4.0 * abt_val * (1.0 - abt_val)) + + +def _boundary_weight(latent_mask: torch.Tensor, inpaint_weight: torch.Tensor) -> Optional[torch.Tensor]: + """ + Return a 4-neighbor boundary weight: unknown pixels adjacent to known pixels. + + This replaces the previous dilation-based "ring" (kernel/padding) and has no tunable hyperparameters. + """ + if latent_mask.dim() != 4: + return None + + known = latent_mask > 0.5 + neighbor_known = torch.zeros_like(known) + neighbor_known[:, :, 1:, :] |= known[:, :, :-1, :] + neighbor_known[:, :, :-1, :] |= known[:, :, 1:, :] + neighbor_known[:, :, :, 1:] |= known[:, :, :, :-1] + neighbor_known[:, :, :, :-1] |= known[:, :, :, 1:] + + boundary = (~known) & neighbor_known + return boundary.to(dtype=torch.float32) * inpaint_weight + + +def _weighted_mse(t1: torch.Tensor, t2: torch.Tensor, weight: torch.Tensor) -> float: + diff_sq = (t1.to(dtype=torch.float32) - t2.to(dtype=torch.float32)) ** 2 + denom = torch.sum(weight) + 1e-12 + return float((torch.sum(diff_sq * weight) / denom).item()) + + +class LanPaintEarlyStopper: + """ + Per-step early-stop logic for LanPaint inner (Langevin) iterations. + """ + + @classmethod + def from_options( + cls, + *, + model_options: Optional[dict], + latent_mask: torch.Tensor, + abt: torch.Tensor, + default_threshold: float, + default_patience: int, + default_distance_fn: Optional[Callable[..., Any]], + ) -> Optional["LanPaintEarlyStopper"]: + semantic_stop = model_options.get("lanpaint_semantic_stop") if isinstance(model_options, dict) else None + + threshold = float(default_threshold) + patience = int(default_patience) + distance_fn = default_distance_fn + # distance_fn contract: return None (use default metric) or a scalar (Python number / 0-d (1-element) torch.Tensor) + + if isinstance(semantic_stop, dict): + threshold = float(semantic_stop.get("threshold", threshold)) + patience = int(semantic_stop.get("patience", patience)) + distance_fn = semantic_stop.get("distance_fn", distance_fn) + + # Backward compatibility: map legacy 'min_steps' to a patience floor so it is not an independent knob. + if patience > 0: + min_steps = semantic_stop.get("min_steps") + if min_steps is not None: + try: + min_steps_int = int(min_steps) + except (TypeError, ValueError): + min_steps_int = 0 + if min_steps_int > 1: + patience = max(patience, min_steps_int - 1) + + enabled_early_stop = (threshold > 0.0) and (patience > 0) + # Require N+1 consecutive stable checks: + # - the first stable step sets patience_counter to 1 + # - `patience=1` therefore stops after 2 stable steps + patience_eff = max(1, patience) + 1 + threshold_eff = threshold + inpaint_weight = ring_weight = trace = abt_val = None + + if enabled_early_stop: + try: + abt_val = float(torch.mean(abt).item()) + except (TypeError, ValueError): + abt_val = 0.0 + + threshold_eff = threshold * _abt_scale(abt_val) + if threshold_eff <= 0.0: + enabled_early_stop = False + else: + inpaint_weight = (1 - latent_mask).to(dtype=torch.float32) + if float(torch.sum(inpaint_weight).item()) < 1e-6: + enabled_early_stop = False + else: + ring_weight = _boundary_weight(latent_mask, inpaint_weight) + if isinstance(model_options, dict): + trace = model_options.get("lanpaint_semantic_trace") + + if not enabled_early_stop: + return None + + # Pre-fetch trace keys to avoid repeated dict lookups + bench_case_id = bench_outer_step = bench_timestep = None + if isinstance(trace, list) and isinstance(model_options, dict): + bench_case_id = model_options.get("bench_case_id") + bench_outer_step = model_options.get("bench_outer_step") + bench_timestep = model_options.get("bench_timestep") + + return cls( + enabled=enabled_early_stop, + threshold=threshold, + threshold_eff=threshold_eff, + patience_eff=patience_eff, + inpaint_weight=inpaint_weight, + ring_weight=ring_weight, + distance_fn=distance_fn, + trace=trace, + bench_case_id=bench_case_id, + bench_outer_step=bench_outer_step, + bench_timestep=bench_timestep, + abt_val=abt_val, + ) + + def __init__( + self, + *, + enabled: bool, + threshold: float, + threshold_eff: float, + patience_eff: int, + inpaint_weight: Optional[torch.Tensor], + ring_weight: Optional[torch.Tensor], + distance_fn: Optional[Callable[..., Any]] = None, + trace: Optional[list] = None, + bench_case_id: Any = None, + bench_outer_step: Any = None, + bench_timestep: Any = None, + abt_val: Optional[float] = None, + ) -> None: + self.enabled = bool(enabled) + self.threshold = float(threshold) + self.threshold_eff = float(threshold_eff) + self.patience_eff = int(patience_eff) + + self.inpaint_weight = inpaint_weight + self.ring_weight = ring_weight + + self.trace = trace + self.bench_case_id = bench_case_id + self.bench_outer_step = bench_outer_step + self.bench_timestep = bench_timestep + self.abt_val = abt_val + + self.patience_counter = 0 + self.x0_anchor = None + + self._dist_wrapper = self._wrap_distance_fn(distance_fn) if self.enabled else None + + @property + def has_custom_distance_fn(self) -> bool: + return self._dist_wrapper is not None + + @staticmethod + def _wrap_distance_fn(distance_fn: Optional[Callable[..., Any]]): + """ + Wrap a user-provided `distance_fn` into a normalized callable: fn(prev, cur, ctx) -> dist|None. + + Supported signatures: + - 3+ positional (or *args): `distance_fn(prev, cur, ctx)` + - explicit / **kwargs ctx: `distance_fn(prev, cur, ctx=ctx)` + - default 2-arg: `distance_fn(cur, prev)` + + Return contract: None (use default metric) or a scalar (Python number / 0-d (1-element) torch.Tensor). + """ + if not callable(distance_fn): + return None + + try: + sig = inspect.signature(distance_fn) + params = list(sig.parameters.values()) + + has_ctx_param = "ctx" in sig.parameters + has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params) + has_var_pos = any(p.kind == inspect.Parameter.VAR_POSITIONAL for p in params) + + pos_params = [ + p + for p in params + if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + + if len(pos_params) >= 3 or has_var_pos: + # 3-arg positional: fn(prev, cur, ctx) + return lambda p, c, ctx: distance_fn(p, c, ctx) + if has_ctx_param or has_var_kw: + # keyword ctx: fn(prev, cur, ctx=ctx) + return lambda p, c, ctx: distance_fn(p, c, ctx=ctx) + + # Default 2-arg: fn(cur, prev) + return lambda p, c, ctx: distance_fn(c, p) + except (ValueError, TypeError): + # Fallback for built-ins or complex callables. + def fallback_wrapper(p, c, ctx): + try: + return distance_fn(p, c, ctx) + except TypeError as e: + tb = e.__traceback__ + if tb is not None and tb.tb_frame.f_code is not fallback_wrapper.__code__: + raise + return distance_fn(c, p) + + return fallback_wrapper + + def step( + self, + *, + i: int, + n_steps: int, + x_t_before: torch.Tensor, + x_t_after: torch.Tensor, + x_t_prev_for_custom: Optional[torch.Tensor], + prev_args: Any, + args: Any, + ctx: dict, + ) -> bool: + if not self.enabled: + return False + + # 'inpaint_weight' is guaranteed to be set when enabled is True in the caller. + inpaint = self.inpaint_weight + if inpaint is None: + return False + + dist = None + custom_dist = False + dist_inpaint = dist_ring = dist_drift = x0_prev = x0_cur = None + + if self._dist_wrapper is not None: + dist = self._dist_wrapper(x_t_prev_for_custom, x_t_after, ctx) + if dist is not None: + if isinstance(dist, torch.Tensor): + if dist.numel() != 1: + raise TypeError("distance_fn must return None or a scalar / 0-d (1-element) tensor") + dist = float(dist.item()) + else: + dist = float(dist) + custom_dist = dist is not None + + if dist is None: + def _get_x0(arg: Any) -> Optional[torch.Tensor]: + if isinstance(arg, LangevinState): + return arg.x0 + if isinstance(arg, tuple) and len(arg) >= 3: + return arg[2] + return None + + x0_prev = _get_x0(prev_args) + x0_cur = _get_x0(args) + + if x0_prev is not None and x0_cur is not None: + dist_inpaint = _weighted_mse(x0_cur, x0_prev, inpaint) + dist_ring = _weighted_mse(x0_cur, x0_prev, self.ring_weight) if self.ring_weight is not None else None + dist = dist_inpaint if dist_ring is None else max(dist_inpaint, dist_ring) + else: + dist_inpaint = _weighted_mse(x_t_after, x_t_before, inpaint) + dist = dist_inpaint + + threshold_used = self.threshold if custom_dist else self.threshold_eff + + # Drift guard (only for default metric with x0_cur). + if x0_cur is not None and not custom_dist: + if dist <= threshold_used: + if self.x0_anchor is None: + self.x0_anchor = x0_cur.detach() + else: + drift_inpaint = _weighted_mse(x0_cur, self.x0_anchor, inpaint) + drift_ring = _weighted_mse(x0_cur, self.x0_anchor, self.ring_weight) if self.ring_weight is not None else None + dist_drift = drift_inpaint if drift_ring is None else max(drift_inpaint, drift_ring) + dist = max(dist, dist_drift) + else: + self.x0_anchor = None + + if dist <= threshold_used: + self.patience_counter += 1 + else: + self.patience_counter = 0 + self.x0_anchor = None + + should_stop = self.patience_counter >= self.patience_eff + + if isinstance(self.trace, list): + self.trace.append( + { + "case_id": self.bench_case_id, + "outer_step": self.bench_outer_step, + "bench_timestep": self.bench_timestep, + "inner_step": i + 1, + "dist": dist, + "dist_inpaint": None if dist_inpaint is None else float(dist_inpaint), + "dist_ring": None if dist_ring is None else float(dist_ring), + "dist_drift": None if dist_drift is None else float(dist_drift), + "threshold": float(threshold_used), + "threshold_eff": float(self.threshold_eff), + "patience_counter": int(self.patience_counter), + "patience_eff": int(self.patience_eff), + "abt": None if self.abt_val is None else float(self.abt_val), + "custom_dist": bool(custom_dist), + "stopped": bool(should_stop), + } + ) + + return bool(should_stop) + diff --git a/LanPaint/src/LanPaint/lanpaint.py b/LanPaint/src/LanPaint/lanpaint.py new file mode 100644 index 0000000000000000000000000000000000000000..70b38d54abda0771d0f4c13a645b55f28fefe126 --- /dev/null +++ b/LanPaint/src/LanPaint/lanpaint.py @@ -0,0 +1,272 @@ +import torch +from .utils import StochasticHarmonicOscillator +from functools import partial +from .earlystop import LanPaintEarlyStopper +from .types import LangevinState + +class LanPaint(): + def __init__(self, Model, NSteps, Friction, Lambda, Beta, StepSize, IS_FLUX = False, IS_FLOW = False, EarlyStopThreshold = 0.0, EarlyStopPatience = 1, EarlyStopHook = None): + self.n_steps = NSteps + self.chara_lamb = Lambda + self.IS_FLUX = IS_FLUX + self.IS_FLOW = IS_FLOW + self.step_size = StepSize + self.inner_model = Model + self.friction = Friction + self.chara_beta = Beta + self.img_dim_size = None + self.early_stop_threshold = EarlyStopThreshold + self.early_stop_patience = EarlyStopPatience + self.early_stop_hook = EarlyStopHook + + def add_none_dims(self, array): + # Create a tuple with ':' for the first dimension and 'None' repeated num_nones times + index = (slice(None),) + (None,) * (self.img_dim_size-1) + return array[index] + def remove_none_dims(self, array): + # Create a tuple with ':' for the first dimension and 'None' repeated num_nones times + index = (slice(None),) + (0,) * (self.img_dim_size-1) + return array[index] + def __call__(self, x, latent_image, noise, sigma, latent_mask, current_times, model_options, seed, n_steps=None): + self.img_dim_size = len(x.shape) + self.latent_image = latent_image + self.noise = noise + if torch.mean(torch.abs(self.noise)) < 1e-8: + self.noise = torch.randn_like(self.noise) + if n_steps is None: + n_steps = self.n_steps + return self.LanPaint(x, sigma, latent_mask, current_times, n_steps, model_options, seed, self.IS_FLUX, self.IS_FLOW) + def LanPaint(self, x, sigma, latent_mask, current_times, n_steps, model_options, seed, IS_FLUX, IS_FLOW): + input_x = x + VE_Sigma, abt, Flow_t = current_times + + step_size = self.step_size * (1 - abt) + step_size = self.add_none_dims(step_size) + # self.inner_model.inner_model.scale_latent_inpaint returns variance exploding x_t values + # This is the replace step + def scale_latent_inpaint(x, sigma, noise, latent_image): + return self.inner_model.inner_model.model_sampling.noise_scaling(sigma.reshape([sigma.shape[0]] + [1] * (len(noise.shape) - 1)), noise, latent_image) + + x = x * (1 - latent_mask) + scale_latent_inpaint(x=x, sigma=sigma, noise=self.noise, latent_image=self.latent_image)* latent_mask + + if IS_FLUX or IS_FLOW: + x_t = x * ( self.add_none_dims(abt)**0.5 + (1-self.add_none_dims(abt))**0.5 ) + else: + x_t = x / ( 1+self.add_none_dims(VE_Sigma)**2 )**0.5 # switch to variance perserving x_t values + + ############ LanPaint Iterations Start ############### + # after noise_scaling, noise = latent_image + noise * sigma, which is x_t in the variance exploding diffusion model notation for the known region. + args = None + stopper = LanPaintEarlyStopper.from_options( + model_options=model_options if isinstance(model_options, dict) else None, + latent_mask=latent_mask, + abt=abt, + default_threshold=self.early_stop_threshold, + default_patience=self.early_stop_patience, + default_distance_fn=self.early_stop_hook, + ) + + for i in range(n_steps): + score_func = partial( self.score_model, y = self.latent_image, mask = latent_mask, abt = self.add_none_dims(abt), sigma = self.add_none_dims(VE_Sigma), tflow = self.add_none_dims(Flow_t), model_options = model_options, seed = seed ) + + prev_args = args + x_t_prev = x_t.detach() if (stopper is not None and stopper.has_custom_distance_fn) else None + x_t_before = x_t if (stopper is not None and stopper.enabled) else None + + x_t, args = self.langevin_dynamics(x_t, score_func , latent_mask, step_size , current_times, sigma_x = self.add_none_dims(self.sigma_x(abt)), sigma_y = self.add_none_dims(self.sigma_y(abt)), args = args) + + if stopper is not None: + ctx = { + "step": i, + "steps_done": i + 1, + "n_steps": n_steps, + "mask": latent_mask, + "latent_image": self.latent_image, + "current_times": current_times, + "seed": seed, + } + if stopper.step( + i=i, + n_steps=n_steps, + x_t_before=x_t_before, + x_t_after=x_t, + x_t_prev_for_custom=x_t_prev, + prev_args=prev_args, + args=args, + ctx=ctx, + ): + break + + if IS_FLUX or IS_FLOW: + x = x_t / ( self.add_none_dims(abt)**0.5 + (1-self.add_none_dims(abt))**0.5 ) + else: + x = x_t * ( 1+self.add_none_dims(VE_Sigma)**2 )**0.5 # switch to variance perserving x_t values + ############ LanPaint Iterations End ############### + # out is x_0 + + out, _ = self.inner_model(x, sigma, model_options=model_options, seed=seed) + out = out * (1-latent_mask) + self.latent_image * latent_mask + + input_x.copy_(x) + return out + + def score_model(self, x_t, y, mask, abt, sigma, tflow, model_options, seed): + lamb = self.chara_lamb + if self.IS_FLUX or self.IS_FLOW: + # compute t for flow model, with a small epsilon compensating for numerical error. + x = x_t / ( abt**0.5 + (1-abt)**0.5 ) # switch to Gaussian flow matching + x_0, x_0_BIG = self.inner_model(x, self.remove_none_dims(tflow), model_options=model_options, seed=seed) + else: + x = x_t * ( 1+sigma**2 )**0.5 # switch to variance exploding + x_0, x_0_BIG = self.inner_model(x, self.remove_none_dims(sigma), model_options=model_options, seed=seed) + + score_x = -(x_t - x_0) + score_y = - (1 + lamb) * ( x_t - y ) + lamb * (x_t - x_0_BIG) + return score_x * (1 - mask) + score_y * mask + def sigma_x(self, abt): + # the time scale for the x_t update + return abt**0 + def sigma_y(self, abt): + beta = self.chara_beta * abt ** 0 + return beta + + def langevin_dynamics(self, x_t, score, mask, step_size, current_times, sigma_x=1, sigma_y=0, args=None): + if args is not None and not isinstance(args, LangevinState): + if isinstance(args, tuple): + if len(args) == 2: + # Backwards compat: older state was (v, C) without x0. + args = LangevinState(args[0], args[1], None) + elif len(args) >= 3: + args = LangevinState(args[0], args[1], args[2]) + # prepare the step size and time parameters + with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): + step_sizes = self.prepare_step_size(current_times, step_size, sigma_x, sigma_y) + sigma, abt, dtx, dty, Gamma_x, Gamma_y, A_x, A_y, D_x, D_y = step_sizes + # print('mask',mask.device) + if torch.mean(dtx) <= 0.: + return x_t, args + # ------------------------------------------------------------------------- + # Compute the Langevin dynamics update in variance perserving notation + # ------------------------------------------------------------------------- + #x0 = self.x0_evalutation(x_t, score, sigma, args) + #C = abt**0.5 * x0 / (1-abt) + A = A_x * (1-mask) + A_y * mask + D = D_x * (1-mask) + D_y * mask + dt = dtx * (1-mask) + dty * mask + Gamma = Gamma_x * (1-mask) + Gamma_y * mask + + def Coef_C(x_t): + x0 = x_t + score(x_t) + C = (abt**0.5 * x0 - x_t )/ (1-abt) + A * x_t + return C, x0 + def advance_time(x_t, v, dt, Gamma, A, C, D): + dtype = x_t.dtype + with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): + osc = StochasticHarmonicOscillator(Gamma, A, C, D ) + x_t, v = osc.dynamics(x_t, v, dt ) + x_t = x_t.to(dtype) + v = v.to(dtype) + return x_t, v + + def advance_time_overdamped(x_t, dt, A, C, D): + """ + Overdamped (Gamma -> infinity) limit: + dx = -A x dt + C dt + D dW_t + with C treated as constant over this substep. + """ + dtype = x_t.dtype + with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): + A_dt = A * dt + exp_neg = torch.exp(-A_dt) + + eps = 1e-8 + abs_A = torch.abs(A) + # k = (1 - exp(-A dt)) / A -> dt when A -> 0 + k = torch.where(abs_A < eps, dt, (-torch.expm1(-A_dt)) / A) + # k2 = (1 - exp(-2 A dt)) / (2 A) -> dt when A -> 0 + k2 = torch.where(abs_A < eps, dt, (-torch.expm1(-2 * A_dt)) / (2 * A)) + + mean = exp_neg * x_t + k * C + var = (D ** 2) * k2 + noise = torch.randn_like(x_t) * torch.sqrt(torch.clamp(var, min=0.0)) + x_t = mean + noise + return x_t.to(dtype) + + def run_damped(x_t, args): + if args is None: + v = None + C, x0 = Coef_C(x_t) + x_t, v = advance_time(x_t, v, dt, Gamma, A, C, D) + else: + v = args.v + C = args.C + x_t, v = advance_time(x_t, v, dt/2, Gamma, A, C, D) + C_new, x0 = Coef_C(x_t) + v = v + Gamma**0.5 * ( C_new - C) *dt + x_t, v = advance_time(x_t, v, dt/2, Gamma, A, C, D) + C = C_new + # args is (v, C, x0) for the next inner step. + return x_t, LangevinState(v, C, x0) + + def run_overdamped(x_t, args): + if args is None: + C, x0 = Coef_C(x_t) + x_t = advance_time_overdamped(x_t, dt, A, C, D) + else: + C = args.C + x_t = advance_time_overdamped(x_t, dt / 2, A, C, D) + C_new, x0 = Coef_C(x_t) + x_t = x_t + (C_new - C) * dt + x_t = advance_time_overdamped(x_t, dt / 2, A, C, D) + C = C_new + # args is (v, C, x0); v is None in the overdamped fallback. + return x_t, LangevinState(None, C, x0) + + try: + x_t_next, state = run_damped(x_t, args) + + v_next = state.v + if torch.isnan(x_t_next).any() or (v_next is not None and torch.isnan(v_next).any()): + raise ValueError("NaN detected") + + x_t = x_t_next + except Exception: + x_t, state = run_overdamped(x_t, args) + + # args is (v, C, x0); v can be None if we fell back to the overdamped update. + return x_t, state + + def prepare_step_size(self, current_times, step_size, sigma_x, sigma_y): + # ------------------------------------------------------------------------- + # Unpack current times parameters (sigma and abt) + sigma, abt, flow_t = current_times + sigma = self.add_none_dims(sigma) + abt = self.add_none_dims(abt) + # Compute time step (dtx, dty) for x and y branches. + dtx = 2 * step_size * sigma_x + dty = 2 * step_size * sigma_y + + # ------------------------------------------------------------------------- + # Define friction parameter Gamma_hat for each branch. + # Using dtx**0 provides a tensor of the proper device/dtype. + + Gamma_hat_x = self.friction **2 * self.step_size * sigma_x / 0.1 * sigma**0 + Gamma_hat_y = self.friction **2 * self.step_size * sigma_y / 0.1 * sigma**0 + #print("Gamma_hat_x", torch.mean(Gamma_hat_x).item(), "Gamma_hat_y", torch.mean(Gamma_hat_y).item()) + # adjust dt to match denoise-addnoise steps sizes + Gamma_hat_x /= 2. + Gamma_hat_y /= 2. + A_t_x = (1) / ( 1 - abt ) * dtx / 2 + A_t_y = (1+self.chara_lamb) / ( 1 - abt ) * dty / 2 + + + A_x = A_t_x / (dtx/2) + A_y = A_t_y / (dty/2) + Gamma_x = Gamma_hat_x / (dtx/2) + Gamma_y = Gamma_hat_y / (dty/2) + + #D_x = (2 * (1 + sigma**2) )**0.5 + #D_y = (2 * (1 + sigma**2) )**0.5 + D_x = (2 * abt**0 )**0.5 + D_y = (2 * abt**0 )**0.5 + return sigma, abt, dtx/2, dty/2, Gamma_x, Gamma_y, A_x, A_y, D_x, D_y diff --git a/LanPaint/src/LanPaint/nodes.py b/LanPaint/src/LanPaint/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..5c7064605b6540ebb8f2b5c5f43d4aa3a9048bd3 --- /dev/null +++ b/LanPaint/src/LanPaint/nodes.py @@ -0,0 +1,648 @@ +from contextlib import contextmanager +import math +# import nodes.py +import comfy +import nodes +import latent_preview +import torch +from comfy.utils import repeat_to_batch_size +from comfy.samplers import * +from comfy.model_base import ModelType +from .lanpaint import LanPaint +from comfy.model_base import WAN22 +import comfyui_version + +def _version_tuple(value): + return tuple(int(part) if part.isdigit() else 0 for part in value.split(".")) + +COMFYUI_VERSION_060_OR_NEWER = _version_tuple(comfyui_version.__version__) >= (0, 6, 0) + +def reshape_mask(input_mask, output_shape,video_inpainting=False): + dims = len(output_shape) - 2 + print('output shape',output_shape) + scale_mode = "nearest-exact" + print('input mask',input_mask.shape,type(input_mask),torch.max(input_mask),torch.min(input_mask)) + print('target output_shape',output_shape) + print('input_mask.ndim:', input_mask.ndim, 'output_shape len:', len(output_shape)) + + # Handle input mask dimensions + if input_mask.ndim == 2: + input_mask = input_mask.unsqueeze(0).unsqueeze(0) + elif input_mask.ndim == 3: + input_mask = input_mask.unsqueeze(1) + + # Handle 5D output shape (B, C, F, H, W) by ensuring input is 5D + if len(output_shape) == 5 and input_mask.ndim == 4: + if COMFYUI_VERSION_060_OR_NEWER: + input_mask = input_mask.unsqueeze(2) # (B, C, 1, H, W) + + # Handle video case with temporal dimension + if video_inpainting: # Video case: (batch, channels, frames, height, width) + target_frames = output_shape[2] + target_height, target_width = output_shape[-2:] + + print('Video case - input_mask initial shape:', input_mask.shape) + + # First reshape input_mask to have proper dimensions for video processing + # Assume input is (frames, channels, height, width) -> (1, channels, frames, height, width) + ## if comfy version < 0.6.0 + if not COMFYUI_VERSION_060_OR_NEWER: + input_mask = input_mask.permute(1, 0, 2, 3).unsqueeze(0) + print('Video case - input_mask after reshaping:', input_mask.shape) + # Ensure we have the correct 5D shape: (batch, channels, frames, height, width) + batch_size, channels, frames, height, width = input_mask.shape + print('Video case - dimensions: batch_size={}, channels={}, frames={}, height={}, width={}'.format(batch_size, channels, frames, height, width)) + print('Video case - target size:', (target_frames, target_height, target_width)) + + # 3D nearest-exact interpolation: (batch, channels, frames, height, width) -> (batch, channels, target_frames, target_height, target_width) + temp_mask = torch.nn.functional.interpolate( + input_mask, + size=(target_frames, target_height, target_width), + mode=scale_mode, + ) + + # temp_mask is already 5D: (batch, channels, target_frames, target_height, target_width) + mask = temp_mask + print('after mask',mask.shape) + # Handle channel dimension expansion if needed + if mask.shape[1] < output_shape[1]: + mask = mask.repeat(1, output_shape[1], 1, 1, 1)[:, :output_shape[1]] + # Handle batch dimension + mask = repeat_to_batch_size(mask, output_shape[0]) + else: # Original 2D image case + if not COMFYUI_VERSION_060_OR_NEWER: + mask = torch.nn.functional.interpolate(input_mask, size=output_shape[-2:], mode=scale_mode) + else: + mask = torch.nn.functional.interpolate(input_mask, size=output_shape[2:], mode=scale_mode) + if mask.shape[1] < output_shape[1]: + mask = mask.repeat((1, output_shape[1]) + (1,) * dims)[:,:output_shape[1]] + mask = repeat_to_batch_size(mask, output_shape[0]) + + + return mask +def prepare_mask(noise_mask, shape, device,video_inpainting=False): + return reshape_mask(noise_mask, shape,video_inpainting).to(device) +def sampling_function_LanPaint(model, x, timestep, uncond, cond, cond_scale, cond_scale_BIG, model_options={}, seed=None): + if math.isclose(cond_scale, 1.0) and model_options.get("disable_cfg1_optimization", False) == False: + uncond_ = None + else: + uncond_ = uncond + + conds = [cond, uncond_] + out = calc_cond_batch(model, conds, x, timestep, model_options) + + for fn in model_options.get("sampler_pre_cfg_function", []): + args = {"conds":conds, "conds_out": out, "cond_scale": cond_scale, "timestep": timestep, + "input": x, "sigma": timestep, "model": model, "model_options": model_options} + out = fn(args) + + return cfg_function(model, out[0], out[1], cond_scale, x, timestep, model_options=model_options, cond=cond, uncond=uncond_), cfg_function(model, out[0], out[1], cond_scale_BIG, x, timestep, model_options=model_options, cond=cond, uncond=uncond_) + + +class CFGGuider_LanPaint: + def outer_sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None, **kwargs): + print("CFGGuider outer_sample") + self.inner_model, self.conds, self.loaded_models = comfy.sampler_helpers.prepare_sampling(self.model_patcher, noise.shape, self.conds, self.model_options) + device = self.model_patcher.load_device + + if isinstance(self.inner_model, WAN22): + print("WAN22 detected") + self.inner_model.extra_conds = super(WAN22, self.inner_model).extra_conds + + if denoise_mask is not None: + video_inpainting = self.model_options.get("video_inpainting", False) + denoise_mask = prepare_mask(denoise_mask, noise.shape, device, video_inpainting) + + noise = noise.to(device) + latent_image = latent_image.to(device) + sigmas = sigmas.to(device) + cast_to_load_options(self.model_options, device=device, dtype=self.model_patcher.model_dtype()) + + try: + self.model_patcher.pre_run() + output = self.inner_sample(noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, **kwargs) + finally: + self.model_patcher.cleanup() + + comfy.sampler_helpers.cleanup_models(self.conds, self.loaded_models) + del self.inner_model + del self.loaded_models + return output + def predict_noise(self, x, timestep, model_options={}, seed=None): + return sampling_function_LanPaint(self.inner_model, x, timestep, self.conds.get("negative", None), self.conds.get("positive", None), self.cfg, self.cfg_BIG, model_options=model_options, seed=seed) + +#CFGGuider.outer_sample = CFGGuider_LanPaint.outer_sample +#CFGGuider.predict_noise = CFGGuider_LanPaint.predict_noise + +class KSamplerX0Inpaint: + def __init__(self, model, sigmas): + self.inner_model = model + self.sigmas = sigmas + #self.model_sigmas = torch.cat( (torch.tensor([0.], device = sigmas.device) , torch.tensor( self.inner_model.model_patcher.get_model_object("model_sampling").sigmas, device = sigmas.device) ) ) + #self.model_sigmas = torch.tensor( self.model_sigmas, dtype = self.sigmas.dtype ) + def __call__(self, x, sigma, denoise_mask, model_options={}, seed=None,**kwargs): + ### For 1.5 and XL model + # x is x_t in the notation of variance exploding diffusion model, x_t = x_0 + sigma * noise + # sigma is the noise level + ### For flux model + # x is rectified flow x_t = sigma * noise + (1.0 - sigma) * x_0 + + IS_FLUX = self.inner_model.inner_model.model_type == ModelType.FLUX + IS_FLOW = self.inner_model.inner_model.model_type == ModelType.FLOW + #print("model class", type(self.inner_model.inner_model)) + #print("model type", self.inner_model.inner_model.model_type, "IS_FLUX", IS_FLUX, "IS_FLOW", IS_FLOW) + #print("sigma", torch.mean(sigma).item(), torch.min(sigma).item(), torch.max(sigma).item()) + # unify the notations into variance exploding diffusion model + if IS_FLUX or IS_FLOW: + Flow_t = sigma + abt = (1 - Flow_t)**2 / ((1 - Flow_t)**2 + Flow_t**2 ) + VE_Sigma = Flow_t / (1 - Flow_t) + #print("t", torch.mean( sigma ).item(), "VE_Sigma", torch.mean( VE_Sigma ).item()) + + + else: + VE_Sigma = sigma + abt = 1/( 1+VE_Sigma**2 ) + Flow_t = (1-abt)**0.5 / ( (1-abt)**0.5 + abt**0.5 ) + + if denoise_mask is not None: + if "denoise_mask_function" in model_options: + denoise_mask = model_options["denoise_mask_function"](sigma, denoise_mask, extra_options={"model": self.inner_model, "sigmas": self.sigmas}) + + denoise_mask = (denoise_mask > 0.5).float() + + latent_mask = 1 - denoise_mask + current_times = (VE_Sigma, abt, Flow_t) + + current_step = torch.argmin( torch.abs( self.sigmas - torch.mean(sigma) ) ) + total_steps = len(self.sigmas)-1 + + if total_steps - current_step <= self.LanPaint_early_stop: + out = self.PaintMethod(x, self.latent_image, self.noise, sigma, latent_mask, current_times, model_options, seed, n_steps=0) + else: + out = self.PaintMethod(x, self.latent_image, self.noise, sigma, latent_mask, current_times, model_options, seed) + else: + out, _ = self.inner_model(x, sigma, model_options=model_options, seed=seed) + + # Add TAESD preview support - directly use the latent_preview module + current_step = model_options.get("i", kwargs.get("i", 0)) + total_steps = model_options.get("total_steps", 0) + + # Only show preview every few steps to improve performance + if current_step % 2 == 0: + # Directly call the preview callback if it exists + callback = model_options.get("callback", None) + if callback is not None: + callback({"i": current_step, "denoised": out, "x": x}) + + return out + +# Custom sampler class extending ComfyUI's KSAMPLER for LanPaint +class KSAMPLER(comfy.samplers.KSAMPLER): + def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False): + #noise here is a randn noise from comfy.sample.prepare_noise + #latent_image is the latent image as input of the KSampler node. For inpainting, it is the masked latent image. Otherwise it is zero tensor. + extra_args["denoise_mask"] = denoise_mask + model_k = KSamplerX0Inpaint(model_wrap, sigmas) + model_k.latent_image = latent_image + if self.inpaint_options.get("random", False): #TODO: Should this be the default? + generator = torch.manual_seed(extra_args.get("seed", 41) + 1) + model_k.noise = torch.randn(noise.shape, generator=generator, device="cpu").to(noise.dtype).to(noise.device) + else: + model_k.noise = noise + + IS_FLUX = model_wrap.inner_model.model_type == ModelType.FLUX + IS_FLOW = model_wrap.inner_model.model_type == ModelType.FLOW + # unify the notations into variance exploding diffusion model + if IS_FLUX: + model_wrap.cfg_BIG = 1.0 + else: + model_wrap.cfg_BIG = model_wrap.model_patcher.LanPaint_cfg_BIG + noise = model_wrap.inner_model.model_sampling.noise_scaling(sigmas[0], noise, latent_image, self.max_denoise(model_wrap, sigmas)) + + model_k.PaintMethod = LanPaint(model_k.inner_model, + model_wrap.model_patcher.LanPaint_NumSteps, + model_wrap.model_patcher.LanPaint_Friction, + model_wrap.model_patcher.LanPaint_Lambda, + model_wrap.model_patcher.LanPaint_Beta, + model_wrap.model_patcher.LanPaint_StepSize, + IS_FLUX = IS_FLUX, + IS_FLOW = IS_FLOW, + EarlyStopThreshold = getattr(model_wrap.model_patcher, "LanPaint_InnerThreshold", 0.0), + EarlyStopPatience = getattr(model_wrap.model_patcher, "LanPaint_InnerPatience", 1), + EarlyStopHook = extra_args.get("model_options", {}).get("lanpaint_semantic_hook", None)) + model_k.LanPaint_early_stop = model_wrap.model_patcher.LanPaint_EarlyStop + #if not inpainting, after noise_scaling, noise = noise * sigma, which is the noise added to the clean latent image in the variance exploding diffusion model notation. + #if inpainting, after noise_scaling, noise = latent_image + noise * sigma, which is x_t in the variance exploding diffusion model notation for the known region. + k_callback = None + total_steps = len(sigmas) - 1 + if callback is not None: + k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps) + #print("LanPaint KSampler call sampler_function", self.sampler_function) + # The main loop! + #print("##########") + #print("Sampling with ", self.sampler_function) + #print("##########") + samples = self.sampler_function(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **self.extra_options) + #print("LanPaint KSampler end sampler_function") + samples = model_wrap.inner_model.model_sampling.inverse_noise_scaling(sigmas[-1], samples) + return samples + +@contextmanager +def override_sample_function(): + original_outer_sample = comfy.samplers.CFGGuider.outer_sample + comfy.samplers.CFGGuider.outer_sample = CFGGuider_LanPaint.outer_sample + + original_predict_noise = comfy.samplers.CFGGuider.predict_noise + comfy.samplers.CFGGuider.predict_noise = CFGGuider_LanPaint.predict_noise + + original_sample = comfy.samplers.KSAMPLER.sample + comfy.samplers.KSAMPLER.sample = KSAMPLER.sample + + try: + yield + finally: + comfy.samplers.KSAMPLER.sample = original_sample + comfy.samplers.CFGGuider.predict_noise = original_predict_noise + comfy.samplers.CFGGuider.outer_sample = original_outer_sample + + +class LanPaint_UpSale_LatentNoiseMask: + @classmethod + def INPUT_TYPES(s): + return {"required": { "samples": ("LATENT",), + "scale": ("INT", {"default": 2, "min": 2, "max": 8, "step": 1}), + }} + RETURN_TYPES = ("LATENT",) + FUNCTION = "set_mask" + + + CATEGORY = "latent/inpaint" + + def set_mask(self, samples, scale): + s = samples.copy() + samples = s['samples'] + # generate a mask with every scaleth pixel set to 1 + mask = torch.zeros(samples.shape[0], 1, samples.shape[2], samples.shape[3], device=samples.device) + 1 + mask[:, :, ::scale, ::scale] = 0 + s["noise_mask"] = mask + return (s,) + +#KSAMPLER_NAMES = ["euler", "dpmpp_2m", "uni_pc"] +KSAMPLER_NAMES = ["euler","euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral", + "dpm_fast", "dpmpp_sde", "dpmpp_sde_gpu", + "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", + "deis", "res_multistep", "res_multistep_ancestral", + "gradient_estimation", "er_sde", "seeds_2", "seeds_3"] + +class LanPaint_KSampler(): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL", {"tooltip": "The model used for denoising the input latent."}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "tooltip": "The random seed used for creating the noise."}), + "steps": ("INT", {"default": 30, "min": 1, "max": 10000, "tooltip": "The number of steps used in the denoising process."}), + "cfg": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01, "tooltip": "The Classifier-Free Guidance scale balances creativity and adherence to the prompt. Higher values result in images more closely matching the prompt however too high values will negatively impact quality."}), + "sampler_name": (KSAMPLER_NAMES, {"tooltip": "Recommended: euler."}), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS, {"default": "karras", "tooltip": "The scheduler controls how noise is gradually removed to form the image."}), + "positive": ("CONDITIONING", {"tooltip": "The conditioning describing the attributes you want to include in the image."}), + "negative": ("CONDITIONING", {"tooltip": "The conditioning describing the attributes you want to exclude from the image."}), + "latent_image": ("LATENT", {"tooltip": "The latent image to denoise."}), + "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "The amount of denoising applied, lower values will maintain the structure of the initial image allowing for image to image sampling."}), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "The number of steps for the Langevin dynamics, representing the turns of thinking per step."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: emphasis image quality, Prompt First: emphasis prompt following"}), + "LanPaint_Info": ("STRING", {"default": "LanPaint KSampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + "Inpainting_mode": (["🖼️ Image Inpainting", "🎬 Video Inpainting"], {"default": "🖼️ Image Inpainting", "tooltip": "Choose Image mode for photos or Video mode for video frames with temporal consistency"}), + } + } + + RETURN_TYPES = ("LATENT",) + OUTPUT_TOOLTIPS = ("The denoised latent.",) + FUNCTION = "sample" + + CATEGORY = "sampling" + DESCRIPTION = "Uses the provided model, positive and negative conditioning to denoise the latent image." + + def sample(self, model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, LanPaint_NumSteps=5, LanPaint_PromptMode="Image First", LanPaint_Info="",Inpainting_mode="🖼️ Image Inpainting"): + + model.LanPaint_StepSize = 0.2 + model.LanPaint_Lambda = 16.0 + model.LanPaint_Beta = 1. + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = 15. + model.LanPaint_EarlyStop = 1 + model.LanPaint_InnerThreshold = 0.0 + model.LanPaint_InnerPatience = 1 + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = cfg + else: + model.LanPaint_cfg_BIG = 0*cfg - 0.5 + + # Convert inpainting_mode to boolean for video_inpainting + video_inpainting = (Inpainting_mode == "🎬 Video Inpainting") + if not hasattr(model, 'model_options') or model.model_options is None: + model.model_options = {} + model.model_options["video_inpainting"] = video_inpainting + + with override_sample_function(): + return nodes.common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=denoise) +class LanPaint_KSamplerAdvanced: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model": ("MODEL",), + "add_noise": (["enable", "disable"], ), + "noise_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "steps": ("INT", {"default": 30, "min": 1, "max": 10000}), + "cfg": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}), + "sampler_name": (KSAMPLER_NAMES, ), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS, ), + "positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "latent_image": ("LATENT", ), + "start_at_step": ("INT", {"default": 0, "min": 0, "max": 10000}), + "end_at_step": ("INT", {"default": 10000, "min": 0, "max": 10000}), + "return_with_leftover_noise": (["disable", "enable"], ), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "The number of steps for the Langevin dynamics, representing the turns of thinking per step."}), + "LanPaint_Lambda": ("FLOAT", {"default": 16., "min": 0.1, "max": 50.0, "step": 0.1, "round": 0.1, "tooltip": "The bidirectional guidance scale. Higher values align with known regions more closely, but may result in instability."}), + "LanPaint_StepSize": ("FLOAT", {"default": 0.2, "min": 0.0001, "max": 1., "step": 0.01, "round": 0.001, "tooltip": "The step size for the Langevin dynamics. Higher values result in faster convergence but may be unstable."}), + "LanPaint_Beta": ("FLOAT", {"default": 1., "min": 0.0001, "max": 5, "step": 0.1, "round": 0.1, "tooltip": "The step size ratio between masked / unmasked regions. Lower value can compensate high values of LanPaint_Lambda."}), + "LanPaint_Friction": ("FLOAT", {"default": 15, "min": 0., "max": 50.0, "step": 0.1, "round": 0.1, "tooltip": "The friction parameter for fast langevin, lower values result in faster convergence but may be unstable."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: emphasis image quality, Prompt First: emphasis prompt following"}), + "LanPaint_EarlyStop": ("INT", {"default": 1, "min": 0, "max": 10000, "tooltip": "The number of steps to stop the LanPaint early, useful for preventing the image from irregular patterns."}), + "LanPaint_Info": ("STRING", {"default": "LanPaint KSampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + "Inpainting_mode": (["🖼️ Image Inpainting", "🎬 Video Inpainting"], {"default": "🖼️ Image Inpainting", "tooltip": "Choose Image mode for photos or Video mode for video frames with temporal consistency"}), + "LanPaint_InnerThreshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.0001, "round": 0.0001, "tooltip": "Early stop threshold for Langevin iterations based on semantic distance. 0.0 to disable. (Contributed by godnight10061)"}), + "LanPaint_InnerPatience": ("INT", {"default": 1, "min": 1, "max": 100, "tooltip": "Number of consecutive steps below threshold required to stop. (Contributed by godnight10061)"}), + }, + } + + RETURN_TYPES = ("LATENT",) + FUNCTION = "sample" + + CATEGORY = "sampling" + + def sample(self, model, add_noise, noise_seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, start_at_step, end_at_step, return_with_leftover_noise, LanPaint_NumSteps=5, LanPaint_Lambda=16.0, LanPaint_StepSize=0.2, LanPaint_Beta=1.0, LanPaint_Friction=15.0, LanPaint_PromptMode="Image First", LanPaint_EarlyStop=1, LanPaint_Info="", Inpainting_mode="🖼️ Image Inpainting", LanPaint_InnerThreshold=0.0, LanPaint_InnerPatience=1): + force_full_denoise = True + if return_with_leftover_noise == "enable": + force_full_denoise = False + disable_noise = False + if add_noise == "disable": + disable_noise = True + model.LanPaint_StepSize = LanPaint_StepSize + model.LanPaint_Lambda = LanPaint_Lambda + model.LanPaint_Beta = LanPaint_Beta + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = LanPaint_Friction + model.LanPaint_EarlyStop = LanPaint_EarlyStop + model.LanPaint_InnerThreshold = LanPaint_InnerThreshold + model.LanPaint_InnerPatience = LanPaint_InnerPatience + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = cfg + else: + model.LanPaint_cfg_BIG = 0*cfg - 0.5 + + # Convert inpainting_mode to boolean for video_inpainting + video_inpainting = (Inpainting_mode == "🎬 Video Inpainting") + if not hasattr(model, 'model_options') or model.model_options is None: + model.model_options = {} + model.model_options["video_inpainting"] = video_inpainting + + with override_sample_function(): + return nodes.common_ksampler(model, noise_seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, disable_noise=disable_noise, start_step=start_at_step, last_step=end_at_step, force_full_denoise=force_full_denoise) + + +class MaskBlend: + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image1": ("IMAGE", {"tooltip": "Image before inpaint"}), + "image2": ("IMAGE", {"tooltip": "Image after inpaint"}), + "mask": ("MASK",), + "blend_overlap": ("INT", {"default": 1, "min": 1, "max": 51, "step": 2, "tooltip": "The number of pixels to blend between the two images."}) + }, + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "blend_images" + + CATEGORY = "image/postprocessing" + + def blend_images(self, image1: torch.Tensor, image2: torch.Tensor, mask: torch.Tensor, blend_overlap: int): + # smooth the binary 01 mask, keep 1 still 1, but smooth the transition from 1 to 0 + # for each mask pixel, find out the nearest 1 pixel, and set the mask value to the distance between the two pixels + # check the size of mask and image1, image2, if not the same, assert error + if image1.shape[1] != image2.shape[1] or image1.shape[2] != image2.shape[2]: + raise ValueError( + "Image size mismatch: Image1 and Image2 must have the same dimensions.\n" + "Additionally, ensure both images have width and height that are multiples of 8.\n" + "This is required because VAE decode always generates images with dimensions that are multiples of 8.\n" + "If your input images are not multiples of 8, a size mismatch will occur during the decoding process.\n" + "Please resize your images using an image resize node to ensure compatibility.\n" + "Current sizes - Image1: {}x{}, Image2: {}x{}".format( + image1.shape[2], image1.shape[1], image2.shape[2], image2.shape[1] + ) + ) + mask = mask.float() + mask = torch.nn.functional.max_pool2d(mask, kernel_size=blend_overlap, stride=1, padding=blend_overlap//2) + # apply Gaussian blur with kernel size blend_overlap + kernel = self.gaussian_kernel(blend_overlap) + kernel = kernel.to(image1.device) + kernel = kernel[None, None, ...] + + mask = torch.nn.functional.conv2d(mask[:,None,:,:], kernel, padding=blend_overlap//2)[:,0,:,:] + + + blended_image = image1 * (1 - mask[...,None]) + image2 * mask[...,None] + return (blended_image,) + def gaussian_kernel(self,kernel_size): + """ + Creates a 2D Gaussian kernel with the given size and standard deviation (sigma). + """ + sigma = (kernel_size - 1)/4 + # Create a grid of (x, y) coordinates + x = torch.arange(kernel_size).float() - kernel_size // 2 + y = torch.arange(kernel_size).float() - kernel_size // 2 + x_grid, y_grid = torch.meshgrid(x, y, indexing='ij') + + # Compute the Gaussian function + kernel = torch.exp(-(x_grid ** 2 + y_grid ** 2) / (2 * sigma ** 2)) + kernel = kernel / kernel.sum() # Normalize the kernel + + return kernel + +class Noise_EmptyNoise: + def generate_noise(self, latent): + return torch.zeros_like(latent["samples"]) + +class Noise_RandomNoise: + def __init__(self, seed): + self.seed = seed + def generate_noise(self, latent): + torch.manual_seed(self.seed) + return torch.randn_like(latent["samples"]) + +# Custom sampler implementation mimmicking base comfy nodes_custom_sampler.py +class LanPaint_SamplerCustom: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model": ("MODEL",), + "add_noise": ("BOOLEAN", {"default": True}), + "noise_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "control_after_generate": True}), + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "sampler": ("SAMPLER",), + "sigmas": ("SIGMAS",), + "latent_image": ("LATENT",), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "Number of steps for Langevin dynamics, representing turns of thinking per step."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: prioritizes image quality; Prompt First: prioritizes prompt adherence."}), + "LanPaint_Info": ("STRING", {"default": "LanPaint Custom Sampler. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + } + } + + RETURN_TYPES = ("LATENT", "LATENT") + RETURN_NAMES = ("output", "denoised_output") + FUNCTION = "sample" + CATEGORY = "sampling/custom_sampling" + + def sample(self, model, sampler, sigmas, add_noise, noise_seed, cfg, positive, negative, latent_image, LanPaint_NumSteps, LanPaint_PromptMode, LanPaint_Info=""): + model.LanPaint_StepSize = 0.2 + model.LanPaint_Lambda = 16.0 + model.LanPaint_Beta = 1. + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = 15. + model.LanPaint_EarlyStop = 1 + model.LanPaint_InnerThreshold = 0.0 + model.LanPaint_InnerPatience = 1 + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = cfg + else: + model.LanPaint_cfg_BIG = 0 * cfg - 0.5 + with override_sample_function(): + latent = latent_image.copy() + latent_image = latent["samples"] + latent_image = comfy.sample.fix_empty_latent_channels(model, latent_image) + latent["samples"] = latent_image + + if not add_noise: + noise = Noise_EmptyNoise().generate_noise(latent) + else: + noise = Noise_RandomNoise(noise_seed).generate_noise(latent) + + noise_mask = None + if "noise_mask" in latent: + noise_mask = latent["noise_mask"] + + x0_output = {} + callback = latent_preview.prepare_callback(model, sigmas.shape[-1] - 1, x0_output) + disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED + + samples = comfy.sample.sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image,noise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=noise_seed) + + out = latent.copy() + out["samples"] = samples + if "x0" in x0_output: + out_denoised = latent.copy() + out_denoised["samples"] = model.model.process_latent_out(x0_output["x0"].cpu()) + else: + out_denoised = out + return (out, out_denoised) + +class LanPaint_SamplerCustomAdvanced: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"noise": ("NOISE",), + "guider": ("GUIDER", ), + "sampler": ("SAMPLER", ), + "sigmas": ("SIGMAS", ), + "latent_image": ("LATENT", ), + "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "Number of steps for Langevin dynamics, representing turns of thinking per step."}), + "LanPaint_Lambda": ("FLOAT", {"default": 16.0, "min": 0.1, "max": 50.0, "step": 0.1, "tooltip": "Bidirectional guidance scale. Higher values align with known regions but may cause instability."}), + "LanPaint_StepSize": ("FLOAT", {"default": 0.2, "min": 0.0001, "max": 1.0, "step": 0.01, "tooltip": "Step size for Langevin dynamics. Higher values speed convergence but may be unstable."}), + "LanPaint_Beta": ("FLOAT", {"default": 1.0, "min": 0.0001, "max": 5.0, "step": 0.1, "tooltip": "Step size ratio between masked/unmasked regions. Lower values balance high Lambda."}), + "LanPaint_Friction": ("FLOAT", {"default": 15.0, "min": 0.0, "max": 50.0, "step": 0.1, "tooltip": "Friction parameter for fast Langevin. Lower values speed convergence but may be unstable."}), + "LanPaint_PromptMode": (["Image First", "Prompt First"], {"tooltip": "Image First: prioritizes image quality; Prompt First: prioritizes prompt adherence."}), + "LanPaint_EarlyStop": ("INT", {"default": 1, "min": 0, "max": 10000, "tooltip": "Steps to stop LanPaint early, preventing irregular patterns."}), + "LanPaint_Info": ("STRING", {"default": "LanPaint Custom Sampler Adv. For more info, visit https://github.com/scraed/LanPaint. If you find it useful, please give a star ⭐️!", "multiline": True}), + "LanPaint_InnerThreshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.0001, "round": 0.0001, "tooltip": "Early stop threshold for Langevin iterations based on semantic distance. 0.0 to disable. (Contributed by godnight10061)"}), + "LanPaint_InnerPatience": ("INT", {"default": 1, "min": 1, "max": 100, "tooltip": "Number of consecutive steps below threshold required to stop. (Contributed by godnight10061)"}), + } + } + + RETURN_TYPES = ("LATENT","LATENT") + RETURN_NAMES = ("output", "denoised_output") + + FUNCTION = "sample" + + CATEGORY = "sampling/custom_sampling" + + def sample(self, noise, guider, sampler, sigmas, latent_image, LanPaint_NumSteps, LanPaint_Lambda, LanPaint_StepSize, LanPaint_Beta, LanPaint_Friction, LanPaint_PromptMode, LanPaint_EarlyStop, LanPaint_Info="", LanPaint_InnerThreshold=0.0, LanPaint_InnerPatience=1): + model = guider.model_patcher + model.LanPaint_StepSize = LanPaint_StepSize + model.LanPaint_Lambda = LanPaint_Lambda + model.LanPaint_Beta = LanPaint_Beta + model.LanPaint_NumSteps = LanPaint_NumSteps + model.LanPaint_Friction = LanPaint_Friction + model.LanPaint_EarlyStop = LanPaint_EarlyStop + model.LanPaint_InnerThreshold = LanPaint_InnerThreshold + model.LanPaint_InnerPatience = LanPaint_InnerPatience + if LanPaint_PromptMode == "Image First": + model.LanPaint_cfg_BIG = guider.cfg + else: + model.LanPaint_cfg_BIG = 0 * guider.cfg - 0.5 + with override_sample_function(): + latent = latent_image + latent_image = latent["samples"] + latent = latent.copy() + latent_image = comfy.sample.fix_empty_latent_channels(guider.model_patcher, latent_image) + latent["samples"] = latent_image + + noise_mask = None + if "noise_mask" in latent: + noise_mask = latent["noise_mask"] + + x0_output = {} + callback = latent_preview.prepare_callback(guider.model_patcher, sigmas.shape[-1] - 1, x0_output) + + disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED + samples = guider.sample(noise.generate_noise(latent), latent_image, sampler, sigmas, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=noise.seed) + samples = samples.to(comfy.model_management.intermediate_device()) + + out = latent.copy() + out["samples"] = samples + if "x0" in x0_output: + out_denoised = latent.copy() + out_denoised["samples"] = guider.model_patcher.model.process_latent_out(x0_output["x0"].cpu()) + else: + out_denoised = out + return (out, out_denoised) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "LanPaint_KSampler": LanPaint_KSampler, + "LanPaint_KSamplerAdvanced": LanPaint_KSamplerAdvanced, + "LanPaint_SamplerCustom" : LanPaint_SamplerCustom, + "LanPaint_SamplerCustomAdvanced" : LanPaint_SamplerCustomAdvanced, + "LanPaint_MaskBlend": MaskBlend, +# "LanPaint_UpSale_LatentNoiseMask": LanPaint_UpSale_LatentNoiseMask, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "LanPaint_KSampler": "LanPaint KSampler", + "LanPaint_KSamplerAdvanced": "LanPaint KSampler (Advanced)", + "LanPaint_SamplerCustom" : "LanPaint Sampler Custom", + "LanPaint_SamplerCustomAdvanced" : "LanPaint Sampler Custom (Advanced)", + "LanPaint_MaskBlend": "LanPaint Mask Blend", +# "LanPaint_UpSale_LatentNoiseMask": "LanPaint UpSale Latent Noise Mask" +} diff --git a/LanPaint/src/LanPaint/types.py b/LanPaint/src/LanPaint/types.py new file mode 100644 index 0000000000000000000000000000000000000000..a738a857d08623d64fe153aed731c56440bd9cb0 --- /dev/null +++ b/LanPaint/src/LanPaint/types.py @@ -0,0 +1,10 @@ +from typing import NamedTuple, Optional + +import torch + + +class LangevinState(NamedTuple): + v: Optional[torch.Tensor] + C: Optional[torch.Tensor] + x0: Optional[torch.Tensor] + diff --git a/LanPaint/src/LanPaint/utils.py b/LanPaint/src/LanPaint/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..774ad4c7bd4980a27e6d892f7a5ed9ae70c03e25 --- /dev/null +++ b/LanPaint/src/LanPaint/utils.py @@ -0,0 +1,300 @@ +import torch +def epxm1_x(x): + # Compute the (exp(x) - 1) / x term with a small value to avoid division by zero. + result = torch.special.expm1(x) / x + # replace NaN or inf values with 0 + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + mask = torch.abs(x) < 1e-2 + result = torch.where(mask, 1 + x/2. + x**2 / 6., result) + return result +def epxm1mx_x2(x): + # Compute the (exp(x) - 1 - x) / x**2 term with a small value to avoid division by zero. + result = (torch.special.expm1(x) - x) / x**2 + # replace NaN or inf values with 0 + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + mask = torch.abs(x**2) < 1e-2 + result = torch.where(mask, 1/2. + x/6 + x**2 / 24 + x**3 / 120, result) + return result + +def expm1mxmhx2_x3(x): + # Compute the (exp(x) - 1 - x - x**2 / 2) / x**3 term with a small value to avoid division by zero. + result = (torch.special.expm1(x) - x - x**2 / 2) / x**3 + # replace NaN or inf values with 0 + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + mask = torch.abs(x**3) < 1e-2 + result = torch.where(mask, 1/6 + x/24 + x**2 / 120 + x**3 / 720 + x**4 / 5040, result) + return result + +def exp_1mcosh_GD(gamma_t, delta): + """ + Compute e^(-Γt) * (1 - cosh(Γt√Δ))/ ( (Γt)**2 Δ ) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + # Main computation + is_positive = delta > 0 + sqrt_abs_delta = torch.sqrt(torch.abs(delta)) + gamma_t_sqrt_delta = gamma_t * sqrt_abs_delta + numerator_pos = torch.exp(-gamma_t) - (torch.exp(gamma_t * (sqrt_abs_delta - 1)) + torch.exp(gamma_t * (-sqrt_abs_delta - 1))) / 2 + numerator_neg = torch.exp(-gamma_t) * ( 1 - torch.cos(gamma_t * sqrt_abs_delta ) ) + numerator = torch.where(is_positive, numerator_pos, numerator_neg) + result = numerator / (delta * gamma_t**2 ) + # Handle NaN/inf cases + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + # Handle numerical instability for small delta + mask = torch.abs(gamma_t_sqrt_delta**2) < 5e-2 + taylor = ( -0.5 - gamma_t**2 / 24 * delta - gamma_t**4 / 720 * delta**2 ) * torch.exp(-gamma_t) + result = torch.where(mask, taylor, result) + return result + +def exp_sinh_GsqrtD(gamma_t, delta): + """ + Compute e^(-Γt) * sinh(Γt√Δ) / (Γt√Δ) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + # Main computation + is_positive = delta > 0 + sqrt_abs_delta = torch.sqrt(torch.abs(delta)) + gamma_t_sqrt_delta = gamma_t * sqrt_abs_delta + numerator_pos = (torch.exp(gamma_t * (sqrt_abs_delta - 1)) - torch.exp(gamma_t * (-sqrt_abs_delta - 1))) / 2 + result_pos = numerator_pos / gamma_t_sqrt_delta + result_pos = torch.where(torch.isfinite(result_pos), result_pos, torch.zeros_like(result_pos)) + + # Taylor expansion for small gamma_t_sqrt_delta + mask = torch.abs(gamma_t_sqrt_delta) < 1e-2 + taylor = ( 1 + gamma_t**2 / 6 * delta + gamma_t**4 / 120 * delta**2 ) * torch.exp(-gamma_t) + result_pos = torch.where(mask, taylor, result_pos) + + # Handle negative delta + result_neg = torch.exp(-gamma_t) * torch.special.sinc(gamma_t_sqrt_delta/torch.pi) + result = torch.where(is_positive, result_pos, result_neg) + return result + +def exp_cosh(gamma_t, delta): + """ + Compute e^(-Γt) * cosh(Γt√Δ) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + exp_1mcosh_GD_result = exp_1mcosh_GD(gamma_t, delta) # e^(-Γt) * (1 - cosh(Γt√Δ))/ ( (Γt)**2 Δ ) + result = torch.exp(-gamma_t) - gamma_t**2 * delta * exp_1mcosh_GD_result + return result +def exp_sinh_sqrtD(gamma_t, delta): + """ + Compute e^(-Γt) * sinh(Γt√Δ) / √Δ + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + Returns: + Result of the computation with numerical stability handling + """ + exp_sinh_GsqrtD_result = exp_sinh_GsqrtD(gamma_t, delta) # e^(-Γt) * sinh(Γt√Δ) / (Γt√Δ) + result = gamma_t * exp_sinh_GsqrtD_result + return result + + + +def zeta1(gamma_t, delta): + # Compute hyperbolic terms and exponential + half_gamma_t = gamma_t / 2 + exp_cosh_term = exp_cosh(half_gamma_t, delta) + exp_sinh_term = exp_sinh_sqrtD(half_gamma_t, delta) + + + # Main computation + numerator = 1 - (exp_cosh_term + exp_sinh_term) + denominator = gamma_t * (1 - delta) / 4 + result = 1 - numerator / denominator + + # Handle numerical instability + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + + # Taylor expansion for small x (similar to your epxm1Dx approach) + mask = torch.abs(denominator) < 5e-3 + term1 = epxm1_x(-gamma_t) + term2 = epxm1mx_x2(-gamma_t) + term3 = expm1mxmhx2_x3(-gamma_t) + taylor = term1 + (1/2.+ term1-3*term2)*denominator + (-1/6. + term1/2 - 4 * term2 + 10 * term3) * denominator**2 + result = torch.where(mask, taylor, result) + + return result + +def exp_cosh_minus_terms(gamma_t, delta): + """ + Compute E^(-tΓ) * (Cosh[tΓ] - 1 - (Cosh[tΓ√Δ] - 1)/Δ) / (tΓ(1 - Δ)) + + Parameters: + gamma_t: Γ*t term (could be a scalar or tensor) + delta: Δ term (could be a scalar or tensor) + + Returns: + Result of the computation with numerical stability handling + """ + exp_term = torch.exp(-gamma_t) + # Compute individual terms + exp_cosh_term = exp_cosh(gamma_t, gamma_t**0) - exp_term # E^(-tΓ) (Cosh[tΓ] - 1) term + exp_cosh_delta_term = - gamma_t**2 * exp_1mcosh_GD(gamma_t, delta) # E^(-tΓ) (Cosh[tΓ√Δ] - 1)/Δ term + + #exp_1mcosh_GD e^(-Γt) * (1 - cosh(Γt√Δ))/ ( (Γt)**2 Δ ) + # Main computation + numerator = exp_cosh_term - exp_cosh_delta_term + denominator = gamma_t * (1 - delta) + + result = numerator / denominator + + # Handle numerical instability + result = torch.where(torch.isfinite(result), result, torch.zeros_like(result)) + + # Taylor expansion for small gamma_t and delta near 1 + mask = (torch.abs(denominator) < 1e-1) + exp_1mcosh_GD_term = exp_1mcosh_GD(gamma_t, delta**0) + taylor = ( + gamma_t*exp_1mcosh_GD_term + 0.5 * gamma_t * exp_sinh_GsqrtD(gamma_t, delta**0) + - denominator / 4 * ( 0.5 * exp_cosh(gamma_t, delta**0) - 4 * exp_1mcosh_GD_term - 5 /2 * exp_sinh_GsqrtD(gamma_t, delta**0) ) + ) + result = torch.where(mask, taylor, result) + + return result + + +def zeta2(gamma_t, delta): + half_gamma_t = gamma_t / 2 + return exp_sinh_GsqrtD(half_gamma_t, delta) + +def sig11(gamma_t, delta): + return 1 - torch.exp(-gamma_t) + gamma_t**2 * exp_1mcosh_GD(gamma_t, delta) + exp_sinh_sqrtD(gamma_t, delta) + + +def Zcoefs(gamma_t, delta): + Zeta1 = zeta1(gamma_t, delta) + Zeta2 = zeta2(gamma_t, delta) + + sq_total = 1 - Zeta1 + gamma_t * (delta - 1) * (Zeta1 - 1)**2 / 8 + amplitude = torch.sqrt(sq_total) + Zcoef1 = ( gamma_t**0.5 * Zeta2 / 2 **0.5 ) / amplitude + Zcoef2 = Zcoef1 * gamma_t *( - 2 * exp_1mcosh_GD(gamma_t, delta) / sig11(gamma_t, delta) ) ** 0.5 + #cterm = exp_cosh_minus_terms(gamma_t, delta) + #sterm = exp_sinh_sqrtD(gamma_t, delta**0) + exp_sinh_sqrtD(gamma_t, delta) + #Zcoef3 = 2 * torch.sqrt( cterm / ( gamma_t * (1 - delta) * cterm + sterm ) ) + Zcoef3 = torch.sqrt( torch.maximum(1 - Zcoef1**2 - Zcoef2**2, sq_total.new_zeros(sq_total.shape)) ) + + return Zcoef1 * amplitude, Zcoef2 * amplitude, Zcoef3 * amplitude, amplitude + +def Zcoefs_asymp(gamma_t, delta): + A_t = (gamma_t * (1 - delta) )/4 + return epxm1_x(- 2 * A_t) + +class StochasticHarmonicOscillator: + """ + Simulates a stochastic harmonic oscillator governed by the equations: + dy(t) = q(t) dt + dq(t) = -Γ A y(t) dt + Γ C dt + Γ D dw(t) - Γ q(t) dt + + Also define v(t) = q(t) / √Γ, which is numerically more stable. + + Where: + y(t) - Position variable + q(t) - Velocity variable + Γ - Damping coefficient + A - Harmonic potential strength + C - Constant force term + D - Noise amplitude + dw(t) - Wiener process (Brownian motion) + """ + def __init__(self, Gamma, A, C, D): + self.Gamma = Gamma + self.A = A + self.C = C + self.D = D + self.Delta = 1 - 4 * A / Gamma + def sig11(self, gamma_t, delta): + return 1 - torch.exp(-gamma_t) + gamma_t**2 * exp_1mcosh_GD(gamma_t, delta) + exp_sinh_sqrtD(gamma_t, delta) + def sig22(self, gamma_t, delta): + return 1- zeta1(2*gamma_t, delta) + 2 * gamma_t * exp_1mcosh_GD(gamma_t, delta) + def dynamics(self, y0, v0, t): + """ + Calculates the position and velocity variables at time t. + + Parameters: + y0 (float): Initial position + v0 (float): Initial velocity v(0) = q(0) / √Γ + t (float): Time at which to evaluate the dynamics + Returns: + tuple: (y(t), v(t)) + """ + + dummyzero = y0.new_zeros(1) # convert scalar to tensor with same device and dtype as y0 + Delta = self.Delta + dummyzero + Gamma_hat = self.Gamma * t + dummyzero + A = self.A + dummyzero + C = self.C + dummyzero + D = self.D + dummyzero + Gamma = self.Gamma + dummyzero + zeta_1 = zeta1( Gamma_hat, Delta) + zeta_2 = zeta2( Gamma_hat, Delta) + EE = 1 - Gamma_hat * zeta_2 + + if v0 is None: + v0 = torch.randn_like(y0) * D / 2 ** 0.5 + #v0 = (C - A * y0)/Gamma**0.5 + + # Calculate mean position and velocity + term1 = (1 - zeta_1) * (C * t - A * t * y0) + zeta_2 * (Gamma ** 0.5) * v0 * t + y_mean = term1 + y0 + v_mean = (1 - EE)*(C - A * y0) / (Gamma ** 0.5) + (EE - A * t * (1 - zeta_1)) * v0 + + cov_yy = D**2 * t * self.sig22(Gamma_hat, Delta) + cov_vv = D**2 * self.sig11(Gamma_hat, Delta) / 2 + cov_yv = (zeta2(Gamma_hat, Delta) * Gamma_hat * D ) **2 / 2 / (Gamma ** 0.5) + + # sample new position and velocity with multivariate normal distribution + + batch_shape = y0.shape + cov_matrix = torch.zeros(*batch_shape, 2, 2, device=y0.device, dtype=y0.dtype) + cov_matrix[..., 0, 0] = cov_yy + cov_matrix[..., 0, 1] = cov_yv + cov_matrix[..., 1, 0] = cov_yv # symmetric + cov_matrix[..., 1, 1] = cov_vv + + + + # Compute the Cholesky decomposition to get scale_tril + #scale_tril = torch.linalg.cholesky(cov_matrix) + scale_tril = torch.zeros(*batch_shape, 2, 2, device=y0.device, dtype=y0.dtype) + tol = 1e-8 + cov_yy = torch.clamp( cov_yy, min = tol ) + sd_yy = torch.sqrt( cov_yy ) + inv_sd_yy = 1/(sd_yy) + + scale_tril[..., 0, 0] = sd_yy + scale_tril[..., 0, 1] = 0. + scale_tril[..., 1, 0] = cov_yv * inv_sd_yy + scale_tril[..., 1, 1] = torch.clamp( cov_vv - cov_yv**2 / cov_yy, min = tol ) ** 0.5 + # check if it matches torch.linalg. + #assert torch.allclose(torch.linalg.cholesky(cov_matrix), scale_tril, atol = 1e-4, rtol = 1e-4 ) + # Sample correlated noise from multivariate normal + mean = torch.zeros(*batch_shape, 2, device=y0.device, dtype=y0.dtype) + mean[..., 0] = y_mean + mean[..., 1] = v_mean + new_yv = torch.distributions.MultivariateNormal( + loc=mean, + scale_tril=scale_tril + ).sample() + + return new_yv[...,0], new_yv[...,1] diff --git a/LanPaint/tests/__init__.py b/LanPaint/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..717f0f7eee9420b5025564f75e883f0722b3f23f --- /dev/null +++ b/LanPaint/tests/__init__.py @@ -0,0 +1 @@ +"""Unit test package for LanPaint.""" diff --git a/LanPaint/tests/conftest.py b/LanPaint/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..310609c5ef6b8aff1813f92aca3b954c3b93e9cb --- /dev/null +++ b/LanPaint/tests/conftest.py @@ -0,0 +1,6 @@ +import os +import sys + +# Add the project root directory to Python path +# This allows the tests to import the project +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) diff --git a/LanPaint/tests/pytest.ini b/LanPaint/tests/pytest.ini new file mode 100644 index 0000000000000000000000000000000000000000..e2884105999d8b98a256f564e7eaf393c8b1f52e --- /dev/null +++ b/LanPaint/tests/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +# Keep settings value-only; pytest does not treat inline `# ...` as comments. +testpaths = . +python_files = test_*.py +norecursedirs = .. diff --git a/LanPaint/tests/test_LanPaint.py b/LanPaint/tests/test_LanPaint.py new file mode 100644 index 0000000000000000000000000000000000000000..355dddf06da0e24e154afe942f6767a6e711d3ae --- /dev/null +++ b/LanPaint/tests/test_LanPaint.py @@ -0,0 +1,13 @@ +"""Basic import tests for LanPaint. + +The ComfyUI runtime dependencies (e.g. `comfy`) are intentionally optional for unit tests. +""" + + +def test_package_imports_without_comfy() -> None: + import LanPaint + + assert isinstance(LanPaint.NODE_CLASS_MAPPINGS, dict) + assert isinstance(LanPaint.NODE_DISPLAY_NAME_MAPPINGS, dict) + assert "LanPaint_KSampler" in LanPaint.NODE_CLASS_MAPPINGS + assert LanPaint.WEB_DIRECTORY == "./web" diff --git a/LanPaint/tests/test_lanpaint_semantic_stop.py b/LanPaint/tests/test_lanpaint_semantic_stop.py new file mode 100644 index 0000000000000000000000000000000000000000..e96acbe47e94968878d819b3af8ec0e83dd91bf8 --- /dev/null +++ b/LanPaint/tests/test_lanpaint_semantic_stop.py @@ -0,0 +1,104 @@ +import torch + +from src.LanPaint.lanpaint import LanPaint as LanPaintEngine + + +class _DummySampling: + def noise_scaling(self, sigma, noise, latent_image): # type: ignore[no-untyped-def] + return latent_image + noise * sigma + + +class _DummyModel: + def __init__(self) -> None: + self.inner_model = self + self.model_sampling = _DummySampling() + + def __call__(self, x, sigma, model_options=None, seed=None): # type: ignore[no-untyped-def] + return x, x + + +def _inputs(): # type: ignore[no-untyped-def] + x = torch.zeros((1, 4, 8, 8)) + latent_image = torch.zeros_like(x) + noise = torch.ones_like(x) + sigma = torch.tensor([1.0]) + + latent_mask = torch.zeros_like(x) + current_times = (sigma, torch.tensor([0.5]), torch.tensor([0.0])) + return x, latent_image, noise, sigma, latent_mask, current_times + + +def test_default_semantic_stop_triggers_at_patience_without_custom_distance_fn() -> None: + engine = LanPaintEngine( + _DummyModel(), + NSteps=10, + Friction=15.0, + Lambda=1.0, + Beta=1.0, + StepSize=0.2, + ) + + calls = {"langevin": 0, "with_score": 0, "without_score": 0} + + def fake_langevin(x_t, score, mask, step_size, current_times, sigma_x=1, sigma_y=0, args=None): # type: ignore[no-untyped-def] + calls["langevin"] += 1 + if score is None: + calls["without_score"] += 1 + else: + calls["with_score"] += 1 + return x_t, args + + engine.langevin_dynamics = fake_langevin # type: ignore[method-assign] + + model_options = { + "lanpaint_semantic_stop": { + "threshold": 1e-6, + "patience": 2, + } + } + + x, latent_image, noise, sigma, latent_mask, current_times = _inputs() + engine(x, latent_image, noise, sigma, latent_mask, current_times, model_options=model_options, seed=0, n_steps=10) + + assert calls["langevin"] == 3 + assert calls["with_score"] == 3 + assert calls["without_score"] == 0 + + +def test_semantic_stop_is_disabled_when_no_inpaint_region() -> None: + engine = LanPaintEngine( + _DummyModel(), + NSteps=10, + Friction=15.0, + Lambda=1.0, + Beta=1.0, + StepSize=0.2, + ) + + calls = {"langevin": 0, "with_score": 0, "without_score": 0} + + def fake_langevin(x_t, score, mask, step_size, current_times, sigma_x=1, sigma_y=0, args=None): # type: ignore[no-untyped-def] + calls["langevin"] += 1 + if score is None: + calls["without_score"] += 1 + else: + calls["with_score"] += 1 + return x_t, args + + engine.langevin_dynamics = fake_langevin # type: ignore[method-assign] + + model_options = { + "lanpaint_semantic_stop": { + "threshold": 1e-6, + "patience": 1, + } + } + + x, latent_image, noise, sigma, latent_mask, _ = _inputs() + current_times = (sigma, torch.tensor([0.5]), torch.tensor([0.0])) + no_inpaint_mask = torch.ones_like(latent_mask) + engine(x, latent_image, noise, sigma, no_inpaint_mask, current_times, model_options=model_options, seed=0, n_steps=10) + + assert calls["langevin"] == 10 + assert calls["with_score"] == 10 + assert calls["without_score"] == 0 diff --git a/LanPaint/tests/test_reshape_mask.py b/LanPaint/tests/test_reshape_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..4de8fff31708bea6c5272fb7d49fdff90c74dd20 --- /dev/null +++ b/LanPaint/tests/test_reshape_mask.py @@ -0,0 +1,74 @@ +import importlib +import sys +import types + +import pytest +import torch + + +def _repeat_to_batch_size(tensor: torch.Tensor, batch_size: int) -> torch.Tensor: + if tensor.shape[0] == batch_size: + return tensor + if tensor.shape[0] == 1: + return tensor.repeat((batch_size,) + (1,) * (tensor.ndim - 1)) + repeats = (batch_size + tensor.shape[0] - 1) // tensor.shape[0] + return tensor.repeat((repeats,) + (1,) * (tensor.ndim - 1))[:batch_size] + + +def _import_nodes(monkeypatch, comfyui_version: str): + comfy_mod = types.ModuleType("comfy") + comfy_mod.__path__ = [] + + comfy_utils_mod = types.ModuleType("comfy.utils") + comfy_utils_mod.repeat_to_batch_size = _repeat_to_batch_size + + comfy_samplers_mod = types.ModuleType("comfy.samplers") + class DummyKSAMPLER: ... + comfy_samplers_mod.KSAMPLER = DummyKSAMPLER + + comfy_model_base_mod = types.ModuleType("comfy.model_base") + class ModelType: + FLUX = "FLUX" + FLOW = "FLOW" + + class WAN22: ... + comfy_model_base_mod.ModelType = ModelType + comfy_model_base_mod.WAN22 = WAN22 + + comfyui_version_mod = types.ModuleType("comfyui_version") + comfyui_version_mod.__version__ = comfyui_version + + comfy_mod.utils = comfy_utils_mod + comfy_mod.samplers = comfy_samplers_mod + comfy_mod.model_base = comfy_model_base_mod + + monkeypatch.setitem(sys.modules, "comfy", comfy_mod) + monkeypatch.setitem(sys.modules, "comfy.utils", comfy_utils_mod) + monkeypatch.setitem(sys.modules, "comfy.samplers", comfy_samplers_mod) + monkeypatch.setitem(sys.modules, "comfy.model_base", comfy_model_base_mod) + monkeypatch.setitem(sys.modules, "nodes", types.ModuleType("nodes")) + monkeypatch.setitem(sys.modules, "latent_preview", types.ModuleType("latent_preview")) + monkeypatch.setitem(sys.modules, "comfyui_version", comfyui_version_mod) + + sys.modules.pop("src.LanPaint.nodes", None) + return importlib.import_module("src.LanPaint.nodes") + + +@pytest.mark.parametrize("comfyui_version", ["0.5.0", "0.6.0"]) +def test_reshape_mask_accepts_bhw_and_5d_output_shape(monkeypatch, comfyui_version: str) -> None: + lanpaint_nodes = _import_nodes(monkeypatch, comfyui_version) + input_mask = torch.zeros((1, 4, 4)) + output_shape = (1, 16, 1, 8, 8) + + out = lanpaint_nodes.reshape_mask(input_mask, output_shape, video_inpainting=False) + assert tuple(out.shape) == output_shape + + +def test_prepare_mask_accepts_hw_and_moves_device(monkeypatch) -> None: + lanpaint_nodes = _import_nodes(monkeypatch, "0.5.0") + input_mask = torch.zeros((4, 4)) + output_shape = (2, 3, 8, 8) + + out = lanpaint_nodes.prepare_mask(input_mask, output_shape, device=torch.device("cpu"), video_inpainting=False) + assert tuple(out.shape) == output_shape + assert out.device.type == "cpu" diff --git a/LanPaint/tests/test_sho_regression.py b/LanPaint/tests/test_sho_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..cf46119a096288f1e67c6e2fb4fd19e18c991219 --- /dev/null +++ b/LanPaint/tests/test_sho_regression.py @@ -0,0 +1,45 @@ +import torch +from unittest.mock import MagicMock, patch +from src.LanPaint.lanpaint import LanPaint + + +def test_langevin_dynamics_fallback_on_nan() -> None: + """Test that langevin_dynamics falls back to overdamped dynamics if damped dynamics produces NaNs.""" + torch.manual_seed(0) + # Setup minimal LanPaint instance + lp = LanPaint(Model=MagicMock(), NSteps=10, Friction=1.0, Lambda=1.0, Beta=1.0, StepSize=0.1) + # Dummy inputs + # Shape: (Batch, Channel, Height, Width) + x_t = torch.randn(1, 4, 8, 8) + lp.img_dim_size = 4 + mask = torch.zeros_like(x_t) + # Simple score function + def score(x): + return torch.zeros_like(x) + step_size = torch.tensor([0.1]) + # (sigma, abt, flow_t) + current_times = (torch.tensor([0.5]), torch.tensor([0.5]), torch.tensor([0.5])) + # Mock StochasticHarmonicOscillator to return NaNs + # We patch it where it is used (imported) in lanpaint.py + with patch("src.LanPaint.lanpaint.StochasticHarmonicOscillator") as MockSHO: + mock_instance = MockSHO.return_value + # Configure dynamics to return NaNs + nan_tensor = torch.full_like(x_t, float('nan')) + mock_instance.dynamics.return_value = (nan_tensor, nan_tensor) + # Execute langevin_dynamics + # This should try run_damped -> get NaNs -> raise ValueError -> catch -> run_overdamped + x_out, args_out = lp.langevin_dynamics(x_t, score, mask, step_size, current_times, sigma_y=1.0) + assert hasattr(args_out, "v") + assert hasattr(args_out, "C") + assert hasattr(args_out, "x0") + assert args_out[0] is args_out.v + assert args_out[1] is args_out.C + assert args_out[2] is args_out.x0 + v_out = args_out[0] + # Verify that SHO was initialized and dynamics called + MockSHO.assert_called() + mock_instance.dynamics.assert_called() + # Verify result is finite (indicating fallback to overdamped logic was successful) + assert torch.isfinite(x_out).all(), "Output contains NaNs, fallback failed" + assert v_out is None or torch.isfinite(v_out).all() + diff --git a/LanPaint/web/js/example.js b/LanPaint/web/js/example.js new file mode 100644 index 0000000000000000000000000000000000000000..b2ebc8a1fe71086e51f1542119760d03127ccd8c --- /dev/null +++ b/LanPaint/web/js/example.js @@ -0,0 +1 @@ +console.log(app); diff --git a/VibeVoice-ComfyUI/LICENSE b/VibeVoice-ComfyUI/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9edf0e7689772227f8d9889b78ea6ead1a81d140 --- /dev/null +++ b/VibeVoice-ComfyUI/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Fabio Sarracino - enemyx.net + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/VibeVoice-ComfyUI/README.md b/VibeVoice-ComfyUI/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a61fc4f03302bc0272fc34ecb88ff80f1a2d7be9 --- /dev/null +++ b/VibeVoice-ComfyUI/README.md @@ -0,0 +1,770 @@ +# VibeVoice ComfyUI Nodes + +A comprehensive ComfyUI integration for Microsoft's VibeVoice text-to-speech model, enabling high-quality single and multi-speaker voice synthesis directly within your ComfyUI workflows. + +## ✨ Features + +### Core Functionality +- 🎤 **Single Speaker TTS**: Generate natural speech with optional voice cloning +- 👥 **Multi-Speaker Conversations**: Support for up to 4 distinct speakers +- 🎯 **Voice Cloning**: Clone voices from audio samples +- 🎨 **LoRA Support**: Fine-tune voices with custom LoRA adapters (v1.4.0+) +- 🎚️ **Voice Speed Control**: Adjust speech rate by modifying reference voice speed (v1.5.0+) +- 📝 **Text File Loading**: Load scripts from text files +- 📚 **Automatic Text Chunking**: Handles long texts seamlessly with configurable chunk size +- ⏸️ **Custom Pause Tags**: Insert silences with `[pause]` and `[pause:ms]` tags (wrapper feature) +- 🔄 **Node Chaining**: Connect multiple VibeVoice nodes for complex workflows +- ⏹️ **Interruption Support**: Cancel operations before or between generations +- 🔧 **Flexible Configuration**: Control temperature, sampling, and guidance scale + +### Performance & Optimization +- ⚡ **Attention Mechanisms**: Choose between auto, eager, sdpa, flash_attention_2 or sage +- 🎛️ **Diffusion Steps**: Adjustable quality vs speed trade-off (default: 20) +- 💾 **Memory Management**: Toggle automatic VRAM cleanup after generation +- 🧹 **Free Memory Node**: Manual memory control for complex workflows +- 🍎 **Apple Silicon Support**: Native GPU acceleration on M1/M2/M3 Macs via MPS +- 🔢 **8-Bit Quantization**: Perfect audio quality with high VRAM reduction +- 🔢 **4-Bit Quantization**: Maximum VRAM savings with minimal quality loss + +### Compatibility & Installation +- 📦 **Self-Contained**: Embedded VibeVoice code, no external dependencies +- 🔄 **Universal Compatibility**: Adaptive support for transformers v4.51.3+ +- 🖥️ **Cross-Platform**: Works on Windows, Linux, and macOS +- 🎮 **Multi-Backend**: Supports CUDA, CPU, and MPS (Apple Silicon) + +## 🎥 Video Demo +

+ + VibeVoice ComfyUI Wrapper Demo + +
+ Click to watch the demo video +

+ +## 📦 Installation + +### Automatic Installation (Recommended) +1. Clone this repository into your ComfyUI custom nodes folder: +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/Enemyx-net/VibeVoice-ComfyUI +``` + +2. Restart ComfyUI - the nodes will automatically install requirements on first use + +## 📥 Model Installation + +### Manual Download Required +Starting from version 1.6.0, models and tokenizer must be manually downloaded and placed in the correct folder. The wrapper no longer downloads them automatically. + +### Download Links + +#### Models +You can download VibeVoice models from HuggingFace: + +| Model | Size | Download Link | +|------------------------|--------|---------------| +| **VibeVoice-1.5B** | ~5.4GB | [microsoft/VibeVoice-1.5B](https://huggingface.co/microsoft/VibeVoice-1.5B) | +| **VibeVoice-Large** | ~18.7GB | [aoi-ot/VibeVoice-Large](https://huggingface.co/aoi-ot/VibeVoice-Large) | +| **VibeVoice-Large-Q8** | ~11.6GB | [FabioSarracino/VibeVoice-Large-Q8](https://huggingface.co/FabioSarracino/VibeVoice-Large-Q8) | +| **VibeVoice-Large-Q4** | ~6.6GB | [DevParker/VibeVoice7b-low-vram](https://huggingface.co/DevParker/VibeVoice7b-low-vram) | + +#### Tokenizer (Required) +VibeVoice uses the Qwen2.5-1.5B tokenizer: +- Download from: [Qwen2.5-1.5B Tokenizer](https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main) +- Required files: `tokenizer_config.json`, `vocab.json`, `merges.txt`, `tokenizer.json` + +### Installation Steps +1. Create the models folder if it doesn't exist: + ``` + ComfyUI/models/vibevoice/ + ``` + +2. Download and organize files in the vibevoice folder: + ``` + ComfyUI/models/vibevoice/ + ├── tokenizer/ # Place Qwen tokenizer files here + │ ├── tokenizer_config.json + │ ├── vocab.json + │ ├── merges.txt + │ └── tokenizer.json + ├── VibeVoice-1.5B/ # Model folder + │ ├── config.json + │ ├── model-00001-of-00003.safetensors + │ ├── model-00002-of-00003.safetensors + │ └── ... (other model files) + ├── VibeVoice-Large/ + │ └── ... (model files) + └── my-custom-vibevoice/ # custom names are supported + └── ... (model files) + ``` + +3. For models downloaded from HuggingFace using git-lfs or the HF CLI, you can also use the cache structure: + ``` + ComfyUI/models/vibevoice/ + └── models--microsoft--VibeVoice-1.5B/ + └── snapshots/ + └── [hash]/ + └── ... (model files) + ``` + +4. Refresh your browser - the models will appear in the dropdown menu + +### Notes +- The dropdown will show user-friendly names extracted from folder names +- Both regular folders and HuggingFace cache structures are supported +- Models are rescanned on every browser refresh +- Quantized models are automatically detected from their config files +- The tokenizer is searched in this priority order: + 1. `ComfyUI/models/vibevoice/tokenizer/` (recommended) + 2. `ComfyUI/models/vibevoice/models--Qwen--Qwen2.5-1.5B/` (if exists from previous installations) + 3. HuggingFace cache (if available) + +## 🔧 Available Nodes + +### 1. VibeVoice Load Text From File +Loads text content from files in ComfyUI's input/output/temp directories. +- **Supported formats**: .txt +- **Output**: Text string for TTS nodes + +### 2. VibeVoice Single Speaker +Generates speech from text using a single voice. +- **Text Input**: Direct text or connection from Load Text node +- **Models**: Select from available models in dropdown menu +- **Voice Cloning**: Optional audio input for voice cloning +- **Parameters** (in order): + - `text`: Input text to convert to speech + - `model`: Select from dropdown list of available models found in `ComfyUI/models/vibevoice/` + - `attention_type`: auto, eager, sdpa, flash_attention_2 or sage (default: auto) + - `quantize_llm`: Dynamically quantize only the LLM component for non-quantized models. Options: "full precision" (default), "4bit", or "8bit". 4-bit provides major VRAM savings with minimal quality loss. 8-bit provides a good balance between quality and memory usage. Requires CUDA GPU. Ignored for pre-quantized models. + - `free_memory_after_generate`: Free VRAM after generation (default: True) + - `diffusion_steps`: Number of denoising steps (5-100, default: 20) + - `seed`: Random seed for reproducibility (default: 42) + - `cfg_scale`: Classifier-free guidance (1.0-2.0, default: 1.3) + - `use_sampling`: Enable/disable deterministic generation (default: False) +- **Optional Parameters**: + - `voice_to_clone`: Audio input for voice cloning + - `lora`: LoRA configuration from VibeVoice LoRA node + - `temperature`: Sampling temperature (0.1-2.0, default: 0.95) + - `top_p`: Nucleus sampling parameter (0.1-1.0, default: 0.95) + - `max_words_per_chunk`: Maximum words per chunk for long texts (100-500, default: 250) + - `voice_speed_factor`: Speech rate adjustment (0.8-1.2, default: 1.0, step: 0.01) + +### 3. VibeVoice Multiple Speakers +Generates multi-speaker conversations with distinct voices. +- **Speaker Format**: Use `[N]:` notation where N is 1-4 +- **Voice Assignment**: Optional voice samples for each speaker +- **Recommended Model**: VibeVoice-Large for better multi-speaker quality +- **Parameters** (in order): + - `text`: Input text with speaker labels + - `model`: Select from dropdown list of available models found in `ComfyUI/models/vibevoice/` + - `attention_type`: auto, eager, sdpa, flash_attention_2 or sage (default: auto) + - `quantize_llm`: Dynamically quantize only the LLM component for non-quantized models. Options: "full precision" (default), "4bit", or "8bit". 4-bit provides major VRAM savings with minimal quality loss. 8-bit provides a good balance between quality and memory usage. Requires CUDA GPU. Ignored for pre-quantized models. + - `free_memory_after_generate`: Free VRAM after generation (default: True) + - `diffusion_steps`: Number of denoising steps (5-100, default: 20) + - `seed`: Random seed for reproducibility (default: 42) + - `cfg_scale`: Classifier-free guidance (1.0-2.0, default: 1.3) + - `use_sampling`: Enable/disable deterministic generation (default: False) +- **Optional Parameters**: + - `speaker1_voice` to `speaker4_voice`: Audio inputs for voice cloning + - `lora`: LoRA configuration from VibeVoice LoRA node + - `temperature`: Sampling temperature (0.1-2.0, default: 0.95) + - `top_p`: Nucleus sampling parameter (0.1-1.0, default: 0.95) + - `voice_speed_factor`: Speech rate adjustment for all speakers (0.8-1.2, default: 1.0, step: 0.01) + +### 4. VibeVoice Free Memory +Manually frees all loaded VibeVoice models from memory. +- **Input**: `audio` - Connect audio output to trigger memory cleanup +- **Output**: `audio` - Passes through the input audio unchanged +- **Use Case**: Insert between nodes to free VRAM/RAM at specific workflow points +- **Example**: `[VibeVoice Node] → [Free Memory] → [Save Audio]` + +### 5. VibeVoice LoRA +Configure and load custom LoRA adapters for fine-tuned VibeVoice models. +- **LoRA Selection**: Dropdown menu with available LoRA adapters +- **LoRA Location**: Place your LoRA folders in `ComfyUI/models/vibevoice/loras/` +- **Parameters**: + - `lora_name`: Select from available LoRA adapters or "None" to disable + - `llm_strength`: Strength of the language model LoRA (0.0-2.0, default: 1.0) + - `use_llm`: Apply language model LoRA component (default: True) + - `use_diffusion_head`: Apply diffusion head replacement (default: True) + - `use_acoustic_connector`: Apply acoustic connector LoRA (default: True) + - `use_semantic_connector`: Apply semantic connector LoRA (default: True) +- **Output**: `lora` - LoRA configuration to connect to speaker nodes +- **Usage**: `[VibeVoice LoRA] → [Single/Multiple Speaker Node]` + +## 💬 Multi-Speaker Text Format + +For multi-speaker generation, format your text using the `[N]:` notation: + +``` +[1]: Hello, how are you today? +[2]: I'm doing great, thanks for asking! +[1]: That's wonderful to hear. +[3]: Hey everyone, mind if I join the conversation? +[2]: Not at all, welcome! +``` + +**Important Notes:** +- Use `[1]:`, `[2]:`, `[3]:`, `[4]:` for speaker labels +- Maximum 4 speakers supported +- The system automatically detects the number of speakers from your text +- Each speaker can have an optional voice sample for cloning + +## 🧠 Model Information + +### VibeVoice-1.5B +- **Size**: ~5.4GB download +- **VRAM**: ~6GB +- **Speed**: Faster inference +- **Quality**: Good for single speaker +- **Use Case**: Quick prototyping, single voices + +### VibeVoice-Large +- **Size**: ~18.7GB download +- **VRAM**: ~20GB +- **Speed**: Slower inference but optimized +- **Quality**: Best available quality (full precision) +- **Use Case**: Highest quality production, multi-speaker conversations +- **Note**: Latest official release from Microsoft + +### VibeVoice-Large-Q8 +- **Size**: ~11.6GB download (38% reduction from full model) +- **VRAM**: ~12GB (40% reduction from full precision) +- **Speed**: Balanced inference +- **Quality**: Identical to full precision - perfect audio preservation +- **Use Case**: Production-quality audio with 12GB VRAM GPUs (RTX 3060, 4070 Ti, etc.) +- **Quantization**: Selective 8-bit - only LLM quantized, audio components at full precision +- **Note**: Quantized by Fabio Sarracino + +### VibeVoice-Large-Q4 +- **Size**: ~6.6GB download +- **VRAM**: ~8GB +- **Speed**: Balanced inference +- **Quality**: Good quality with minimal loss +- **Use Case**: Maximum VRAM savings for lower-end GPUs +- **Note**: Quantized by DevParker + +Models are automatically downloaded on first use and cached in `ComfyUI/models/vibevoice/`. + +## ⚙️ Generation Modes + +### Deterministic Mode (Default) +- `use_sampling = False` +- Produces consistent, stable output +- Recommended for production use + +### Sampling Mode +- `use_sampling = True` +- More variation in output +- Uses temperature and top_p parameters +- Good for creative exploration + +## 🎯 Voice Cloning + +To clone a voice: +1. Connect an audio node to the `voice_to_clone` input (single speaker) +2. Or connect to `speaker1_voice`, `speaker2_voice`, etc. (multi-speaker) +3. The model will attempt to match the voice characteristics + +**Requirements for voice samples:** +- Clear audio with minimal background noise +- Minimum 3–10 seconds. Recommended at least 30 seconds for better quality +- Automatically resampled to 24kHz + +## 🎨 LoRA Support + +### Overview +Starting from version 1.4.0, VibeVoice ComfyUI supports custom LoRA (Low-Rank Adaptation) adapters for fine-tuning voice characteristics. This allows you to train and use specialized voice models while maintaining the base VibeVoice capabilities. + +### Setting Up LoRA Adapters + +1. **LoRA Directory Structure**: + Place your LoRA adapter folders in: `ComfyUI/models/vibevoice/loras/` + ``` + ComfyUI/ + └── models/ + └── vibevoice/ + └── loras/ + ├── my_custom_voice/ + │ ├── adapter_config.json + │ ├── adapter_model.safetensors + │ └── diffusion_head/ (optional) + ├── character_voice/ + └── style_adaptation/ + ``` + +2. **Required Files**: + - `adapter_config.json`: LoRA configuration + - `adapter_model.safetensors` or `adapter_model.bin`: Model weights + - Optional components: + - `diffusion_head/`: Custom diffusion head weights + - `acoustic_connector/`: Acoustic connector adaptation + - `semantic_connector/`: Semantic connector adaptation + +### Using LoRA in ComfyUI + +1. **Add VibeVoice LoRA Node**: + - Create a "VibeVoice LoRA" node in your workflow + - Select your LoRA from the dropdown menu + - Configure component settings and strength + +2. **Connect to Speaker Nodes**: + - Connect the LoRA node's output to the `lora` input of speaker nodes + - Both Single Speaker and Multiple Speakers nodes support LoRA + +3. **LoRA Parameters**: + - **llm_strength**: Controls the influence of the language model LoRA (0.0-2.0) + - **Component toggles**: Enable/disable specific LoRA components + - Select "None" to disable LoRA and use the base model + +### Training Your Own LoRA + +To create custom LoRA adapters for VibeVoice, use the official fine-tuning repository: +- **Repository**: [VibeVoice Fine-tuning](https://github.com/voicepowered-ai/VibeVoice-finetuning) +- **Features**: + - Parameter-efficient fine-tuning + - Support for custom datasets + - Adjustable LoRA rank and scaling + - Optional diffusion head adaptation + +### Best Practices + +- **Voice Consistency**: Use the same LoRA across all chunks for long texts +- **Memory Management**: LoRA adds minimal memory overhead (~100-500MB) +- **Compatibility**: LoRA adapters are compatible with all VibeVoice model variants +- **Strength Tuning**: Start with default strength (1.0) and adjust based on results + +### Compatibility Note + +⚠️ **Transformers Version**: The LoRA implementation was developed and tested with `transformers==4.51.3`. While our wrapper supports `transformers>=4.51.3`, LoRA functionality with newer versions of transformers is not guaranteed. If you experience issues with LoRA loading, consider using `transformers==4.51.3` specifically: +```bash +pip install transformers==4.51.3 +``` + +### 🙏 Credits + +LoRA implementation by [@jpgallegoar](https://github.com/jpgallegoar) (PR #127) + +## 🎚️ Voice Speed Control + +### Overview +The Voice Speed Control feature allows you to influence the speaking rate of generated speech by adjusting the speed of the reference voice. This feature modifies the input voice sample before processing, causing the model to learn and reproduce the altered speech rate. + +**Available from version 1.5.0** + +### How It Works +The system applies time-stretching to the reference voice audio: +- Values < 1.0 slow down the reference voice, resulting in slower generated speech +- Values > 1.0 speed up the reference voice, resulting in faster generated speech +- The model learns from the modified voice characteristics and generates speech at a similar pace + +### Usage +- **Parameter**: `voice_speed_factor` +- **Range**: 0.8 to 1.2 +- **Default**: 1.0 (normal speed) +- **Step**: 0.01 (1% increments) + +### Recommended Settings +- **Optimal Range**: 0.95 to 1.05 for natural-sounding results +- **Slower Speech**: Try 0.95 (5% slower) or 0.97 (3% slower) +- **Faster Speech**: Try 1.03 (3% faster) or 1.05 (5% faster) +- **Best Results**: Provide reference audio of at least 20 seconds for more accurate speed matching + +### Important Notes +- The effect works best with longer reference audio samples (20+ seconds recommended) +- Extreme values (< 0.9 or > 1.1) may produce unnatural-sounding speech +- In Multi Speaker mode, the speed adjustment applies to all speakers equally +- Synthetic voices (when no audio is provided) are not affected by this parameter + +### 📖 Examples +``` +# Single Speaker +voice_speed_factor: 0.95 # Slightly slower, more deliberate speech +voice_speed_factor: 1.05 # Slightly faster, more energetic speech + +# Multi Speaker +voice_speed_factor: 0.98 # All speakers talk 2% slower +voice_speed_factor: 1.02 # All speakers talk 2% faster +``` + +## ⏸️ Pause Tags Support + +### Overview +The VibeVoice wrapper includes a custom pause tag feature that allows you to insert silences between text segments. **This is NOT a standard Microsoft VibeVoice feature** - it's an original implementation of our wrapper to provide more control over speech pacing. + +**Available from version 1.3.0** + +### Usage +You can use two types of pause tags in your text: +- `[pause]` - Inserts a 1-second silence (default) +- `[pause:ms]` - Inserts a custom duration silence in milliseconds (e.g., `[pause:2000]` for 2 seconds) + +### 📖 Examples + +#### Single Speaker +``` +Welcome to our presentation. [pause] Today we'll explore artificial intelligence. [pause:500] Let's begin! +``` + +#### Multi-Speaker +``` +[1]: Hello everyone [pause] how are you doing today? +[2]: I'm doing great! [pause:500] Thanks for asking. +[1]: Wonderful to hear! +``` + +### Important Notes + +⚠️ **Context Limitation Warning**: +> **Note: The pause forces the text to be split into chunks. This may worsen the model's ability to understand the context. The model's context is represented ONLY by its own chunk.** + +This means: +- Text before a pause and text after a pause are processed separately +- The model cannot see across pause boundaries when generating speech +- This may affect prosody and intonation consistency +- This may affect prosody and intonation consistency + +### How It Works +1. The wrapper parses your text to find pause tags +2. Text segments between pauses are processed independently +3. Silence audio is generated for each pause duration +4. All audio segments (speech and silence) are concatenated + +### Best Practices +- Use pauses at natural breaking points (end of sentences, paragraphs) +- Avoid pauses in the middle of phrases where context is important +- Test different pause durations to find what sounds most natural + +## 💡 Tips for Best Results + +1. **Text Preparation**: + - Use proper punctuation for natural pauses + - Break long texts into paragraphs + - For multi-speaker, ensure clear speaker transitions + - Use pause tags sparingly to maintain context continuity + +2. **Model Selection**: + - Use 1.5B for quick single-speaker tasks (fastest, ~8GB VRAM) + - Use Large for absolute best quality (~20GB VRAM) + - Use Large-Q8 for production quality with 12GB VRAM (perfect audio, 38% smaller) + - Use Large-Quant-4Bit for maximum VRAM savings (~7GB VRAM) + +3. **Seed Management**: + - Default seed (42) works well for most cases + - Save good seeds for consistent character voices + - Try random seeds if default doesn't work well + +4. **Performance**: + - First run downloads models (5-17GB) + - Subsequent runs use cached models + - GPU recommended for faster inference + +## 💻 System Requirements + +### Hardware +- **Minimum**: 8GB VRAM for VibeVoice-1.5B +- **Recommended**: 17GB+ VRAM for VibeVoice-Large +- **RAM**: 16GB+ system memory + +### Software +- Python 3.8+ +- PyTorch 2.0+ +- CUDA 11.8+ (for GPU acceleration) +- Transformers 4.51.3+ +- ComfyUI (latest version) + +## 🔧 Troubleshooting + +### Installation Issues +- Ensure you're using ComfyUI's Python environment +- Try manual installation if automatic fails +- Restart ComfyUI after installation + +### Generation Issues +- If voices sound unstable, try deterministic mode +- For multi-speaker, ensure text has proper `[N]:` format +- Check that speaker numbers are sequential (1,2,3 not 1,3,5) + +### Memory Issues +- Large model requires ~16GB VRAM +- Use 1.5B model for lower VRAM systems +- Models use bfloat16 precision for efficiency + +## 📖 Examples + +### Single Speaker +``` +Text: "Welcome to our presentation. Today we'll explore the fascinating world of artificial intelligence." +Model: [Select from available models] +cfg_scale: 1.3 +use_sampling: False +``` + +### Two Speakers +``` +[1]: Have you seen the new AI developments? +[2]: Yes, they're quite impressive! +[1]: I think voice synthesis has come a long way. +[2]: Absolutely, it sounds so natural now. +``` + +### Four Speaker Conversation +``` +[1]: Welcome everyone to our meeting. +[2]: Thanks for having us! +[3]: Glad to be here. +[4]: Looking forward to the discussion. +[1]: Let's begin with the agenda. +``` + +## 📊 Performance Benchmarks + +| Model | VRAM Usage | Context Length | Max Audio Duration | +|--------------------|------------|----------------|-------------------| +| VibeVoice-1.5B | ~6GB | 64K tokens | ~90 minutes | +| VibeVoice-Large | ~20GB | 32K tokens | ~45 minutes | +| VibeVoice-Large-Q8 | ~12GB | 32K tokens | ~45 minutes | +| VibeVoice-Large-Q4 | ~8GB | 32K tokens | ~45 minutes | + +## ⚠️ Known Limitations + +- Maximum 4 speakers in multi-speaker mode +- Works best with English and Chinese text +- Some seeds may produce unstable output +- Background music generation cannot be directly controlled + +## 📄 License + +This ComfyUI wrapper is released under the MIT License. See LICENSE file for details. + +**Note**: The VibeVoice model itself is subject to Microsoft's licensing terms: +- VibeVoice is for research purposes only +- Check Microsoft's VibeVoice repository for full model license details + +## 🔗 Links + +- [Original VibeVoice Repository](https://github.com/microsoft/VibeVoice) - Official Microsoft VibeVoice repository (currently unavailable) + +## 🙏 Credits + +- **VibeVoice Model**: Microsoft Research +- **ComfyUI Integration**: Fabio Sarracino +- **Base Model**: Built on Qwen2.5 architecture + +## 💬 Support + +For issues or questions: +1. Check the troubleshooting section +2. Review ComfyUI logs for error messages +3. Ensure VibeVoice is properly installed +4. Open an issue with detailed error information + +## 🤝 Contributing + +Contributions welcome! Please: +1. Test changes thoroughly +2. Follow existing code style +3. Update documentation as needed +4. Submit pull requests with clear descriptions + +## 📝 Changelog + +### Version 1.8.1 +- Forced installation of the bitsandbytes>=0.48.1 library as version 0.48.0 has a critical bug that prevents the Q8 model from working. +- Bug Fixing + +### Version 1.8.0 +- **New Official 8-bit Quantized Model**: VibeVoice-Large-Q8 + - Released on HuggingFace: [FabioSarracino/VibeVoice-Large-Q8](https://huggingface.co/FabioSarracino/VibeVoice-Large-Q8) + - Model size: 11.6GB (38% reduction from 18.7GB full precision) + - VRAM usage: ~12GB (40% reduction from ~20GB) + - **Perfect audio quality**: Identical to full precision model - no quality degradation + - **Selective quantization approach**: audio-critical components (diffusion head, VAE, connectors) kept at full precision + - Optimized for 12GB VRAM GPUs (RTX 3060, 4070 Ti, etc.) + - Solves the common 8-bit "noise problem" by carefully selecting which components to quantize +- **Added 8-bit Dynamic LLM Quantization** + - New "8bit" option in `quantize_llm` parameter for both Single and Multiple Speaker nodes + - Options now: "full precision" (default), "4bit", "8bit" + - Dynamically quantizes only the LLM component for non-quantized models + - Skips all audio-critical components (diffusion_head, acoustic/semantic connectors, tokenizers) + - Provides good balance between quality and VRAM savings + - Requires CUDA GPU and bitsandbytes library + - Automatically ignored for pre-quantized models + +### Version 1.7.0 +- Added dynamic LLM-only 4-bit quantization for non-quantized models + - New `quantize_llm` parameter in both Single and Multiple Speaker nodes + - Options: "full precision" (default) or "4bit" + - Quantizes only the language model component while keeping diffusion head at full precision + - Significantly faster generation with major VRAM savings + - Minimal quality loss compared to full precision + - Requires CUDA GPU for quantization + - Automatically ignored for pre-quantized models + - Uses NF4 (4-bit NormalFloat) quantization type optimized for neural networks + +### Version 1.6.3 +- Fixed tokenizer initialization error + - Resolved `TypeError: expected str, bytes or os.PathLike object, not NoneType` when loading processor + - Added robust fallback mechanism for tokenizer file path resolution + - Improved handling of vocab.json and merges.txt file loading + - Enhanced error handling for edge cases in tokenizer initialization + +### Version 1.6.2 +- Fixed tokenizer loading issue where HuggingFace cache could interfere with local files +- Tokenizer now loads directly from specified path, avoiding cache conflicts +- Added explicit file path loading for better reliability +- Improved logging to show which tokenizer files are being used + +### Version 1.6.1 +- Improved integration by removing HuggingFace unnecessary settings + +### Version 1.6.0 +- **Major Change**: Removed automatic model downloading from HuggingFace + - Models must now be manually downloaded and placed in `ComfyUI/models/vibevoice/` + - Dynamic model dropdown that scans available models on each browser refresh + - Support for custom folder names and HuggingFace cache structure + - Automatic detection of quantized models from config files + - Better user control over model management + - Eliminates authentication issues with private HuggingFace repos +- **Improved Logging System**: + - Optimized logging to reduce console clutter + - Cleaner output for better user experience + +### Version 1.5.0 +- Added Voice Speed Control feature for adjusting speech rate + - New `voice_speed_factor` parameter in both Single and Multi Speaker nodes + - Time-stretching applied to reference audio to influence output speech rate + - Range: 0.8 to 1.2 with 0.01 step increments + - Recommended range: 0.95 to 1.05 for natural results + - Best results with 20+ seconds of reference audio + +### Version 1.4.3 +- Improved LoRA system with better logging and compatibility checks + - Added model compatibility detection to prevent mismatched LoRA loading + - Enhanced debug logging for LoRA component loading process + - Automatic detection and clear error messages for incompatible model-LoRA combinations + - Prevents loading errors when using quantized models with standard LoRAs + - Minor optimizations to LoRA weight loading process + +### Version 1.4.2 +- Bug Fixing + +### Version 1.4.1 +- Fixed HuggingFace authentication error when loading locally cached models + - Resolved 401 authorization errors for already downloaded models + - Node now correctly uses local model snapshots without requiring HuggingFace API authentication + - Prevents unnecessary API calls when models exist in `ComfyUI/models/vibevoice/` + +### Version 1.4.0 +- Added LoRA (Low-Rank Adaptation) support for fine-tuned models + - New "VibeVoice LoRA" node for configuring custom voice adaptations + - Support for language model, diffusion head, and connector adaptations + - Dropdown menu for easy LoRA selection from `ComfyUI/models/vibevoice/loras/` + - Adjustable LoRA strength and component toggles + - Compatible with both Single and Multiple Speaker nodes + - Minimal memory overhead (~100-500MB per LoRA) + - Credits: Implementation by [@jpgallegoar](https://github.com/jpgallegoar) + +### Version 1.3.0 +- Added custom pause tag support for speech pacing control + - New `[pause]` tag for 1-second silence (default) + - New `[pause:ms]` tag for custom duration in milliseconds (e.g., `[pause:2000]` for 2 seconds) + - Works with both Single Speaker and Multiple Speakers nodes + - Automatically splits text at pause points while maintaining voice consistency + - Note: This is a wrapper feature, not part of Microsoft's VibeVoice + +### Version 1.2.5 +- Bug Fixing + +### Version 1.2.4 +- Added automatic text chunking for long texts in Single Speaker node + - Single Speaker node now automatically splits texts longer than 250 words to prevent audio acceleration issues + - New optional parameter `max_words_per_chunk` (range: 100-500 words, default: 250) + - Maintains consistent voice characteristics across all chunks using the same seed + - Seamlessly concatenates audio chunks for smooth, natural output + +### Version 1.2.3 +- Added SageAttention support for inference speedup + - New attention option "sage" using quantized attention (INT8/FP8) for faster generation + - Requirements: NVIDIA GPU with CUDA and sageattention library installation + +### Version 1.2.2 +- Added 4-bit quantized model support + - New model in menu: `VibeVoice-Large-Quant-4Bit` using ~7GB VRAM instead of ~17GB + - Requirements: NVIDIA GPU with CUDA and bitsandbytes library installed + +### Version 1.2.1 +- Bug Fixing + +### Version 1.2.0 +- MPS Support for Apple Silicon: + - Added GPU acceleration support for Mac with Apple Silicon (M1/M2/M3) + - Automatically detects and uses MPS backend when available, providing significant performance improvements over CPU + +### Version 1.1.1 +- Universal Transformers Compatibility: + - Implemented adaptive system that automatically adjusts to different transformers versions + - Guaranteed compatibility from v4.51.3 onwards + - Auto-detects and adapts to API changes between versions + +### Version 1.1.0 +- Updated the URL for downloading the VibeVoice-Large model +- Removed VibeVoice-Large-Preview deprecated model + +### Version 1.0.9 +- Embedded VibeVoice code directly into the wrapper + - Added vvembed folder containing the complete VibeVoice code (MIT licensed) + - No longer requires external VibeVoice installation + - Ensures continued functionality for all users + +### Version 1.0.8 +- BFloat16 Compatibility Fix + - Fixed tensor type compatibility issues with audio processing nodes + - Input audio tensors are now converted from BFloat16 to Float32 for numpy compatibility + - Output audio tensors are explicitly converted to Float32 to ensure compatibility with downstream nodes + - Resolves "Got unsupported ScalarType BFloat16" errors when using voice cloning or saving audio + +### Version 1.0.7 +- Added interruption handler to detect user's cancel request +- Bug fixing + +### Version 1.0.6 +- Fixed a bug that prevented VibeVoice nodes from receiving audio directly from another VibeVoice node + +### Version 1.0.5 +- Added support for Microsoft's official VibeVoice-Large model (stable release) + +### Version 1.0.4 +- Improved tokenizer dependency handling + +### Version 1.0.3 +- Added `attention_type` parameter to both Single Speaker and Multi Speaker nodes for performance optimization + - auto (default): Automatic selection of best implementation + - eager: Standard implementation without optimizations + - sdpa: PyTorch's optimized Scaled Dot Product Attention + - flash_attention_2: Flash Attention 2 for maximum performance (requires compatible GPU) +- Added `diffusion_steps` parameter to control generation quality vs speed trade-off + - Default: 20 (VibeVoice default) + - Higher values: Better quality, longer generation time + - Lower values: Faster generation, potentially lower quality + +### Version 1.0.2 +- Added `free_memory_after_generate` toggle to both Single Speaker and Multi Speaker nodes +- New dedicated "Free Memory Node" for manual memory management in workflows +- Improved VRAM/RAM usage optimization +- Enhanced stability for long generation sessions +- Users can now choose between automatic or manual memory management + +### Version 1.0.1 +- Fixed issue with line breaks in speaker text (both single and multi-speaker nodes) +- Line breaks within individual speaker text are now automatically removed before generation +- Improved text formatting handling for all generation modes + +### Version 1.0.0 +- Initial release +- Single speaker node with voice cloning +- Multi-speaker node with automatic speaker detection +- Text file loading from ComfyUI directories +- Deterministic and sampling generation modes +- Support for VibeVoice 1.5B and Large models \ No newline at end of file diff --git a/VibeVoice-ComfyUI/__init__.py b/VibeVoice-ComfyUI/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f3bcf216146dcbc03e1f9683949960e7b5a4138 --- /dev/null +++ b/VibeVoice-ComfyUI/__init__.py @@ -0,0 +1,121 @@ +# Created by Fabio Sarracino +__version__ = "1.8.1" +__author__ = "Fabio Sarracino" +__title__ = "VibeVoice ComfyUI" + +import logging +import os +import sys +import subprocess + +# Setup logging +logger = logging.getLogger("VibeVoice") +logger.propagate = False + +if not logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter('[VibeVoice] %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + +def apply_timm_compatibility_patches(): + """Apply compatibility patches for timm package conflicts""" + try: + import timm.data + + # Patch missing functions that cause import errors + patches = { + 'ImageNetInfo': lambda: type('ImageNetInfo', (), {'__init__': lambda self: None})(), + 'infer_imagenet_subset': lambda class_to_idx: 'imagenet', + 'get_imagenet_subset_labels': lambda *args, **kwargs: [], + 'get_imagenet_subset_info': lambda *args, **kwargs: {}, + 'resolve_data_config': lambda *args, **kwargs: {} + } + + for attr_name, patch_func in patches.items(): + if not hasattr(timm.data, attr_name): + if attr_name == 'ImageNetInfo': + setattr(timm.data, attr_name, type('ImageNetInfo', (), {'__init__': lambda self: None})) + else: + setattr(timm.data, attr_name, patch_func) + + return True + except Exception as e: + return False + +def check_embedded_vibevoice(): + """Check if embedded VibeVoice is available""" + vvembed_path = os.path.join(os.path.dirname(__file__), 'vvembed') + if not os.path.exists(vvembed_path): + logger.error(f"Embedded VibeVoice not found at {vvembed_path}") + return False + + # Add vvembed to path if not already there + if vvembed_path not in sys.path: + sys.path.insert(0, vvembed_path) + + logger.info("Using embedded VibeVoice (MIT licensed)") + return True + +def ensure_dependencies(): + """Ensure required dependencies are installed""" + try: + import transformers + from packaging import version + if version.parse(transformers.__version__) < version.parse("4.44.0"): + logger.warning("Transformers version < 4.44.0, some features may not work correctly") + except ImportError: + logger.warning("Transformers not installed. Please install: pip install transformers>=4.44.0") + return False + + # Apply timm patches if needed + apply_timm_compatibility_patches() + + return True + +# Initialize node mappings +NODE_CLASS_MAPPINGS = {} +NODE_DISPLAY_NAME_MAPPINGS = {} + +# Register text loading node (always available) +try: + from .nodes.load_text_node import LoadTextFromFileNode + NODE_CLASS_MAPPINGS["LoadTextFromFileNode"] = LoadTextFromFileNode + NODE_DISPLAY_NAME_MAPPINGS["LoadTextFromFileNode"] = "VibeVoice Load Text From File" +except Exception as e: + logger.error(f"Failed to register LoadTextFromFile node: {e}") + +# Register VibeVoice nodes (using embedded VibeVoice) +if check_embedded_vibevoice() and ensure_dependencies(): + try: + from .nodes.single_speaker_node import VibeVoiceSingleSpeakerNode + from .nodes.multi_speaker_node import VibeVoiceMultipleSpeakersNode + from .nodes.free_memory_node import VibeVoiceFreeMemoryNode + from .nodes.lora_node import VibeVoiceLoRANode + + # Single speaker node + NODE_CLASS_MAPPINGS["VibeVoiceSingleSpeakerNode"] = VibeVoiceSingleSpeakerNode + NODE_DISPLAY_NAME_MAPPINGS["VibeVoiceSingleSpeakerNode"] = "VibeVoice Single Speaker" + + # Multi speaker node + NODE_CLASS_MAPPINGS["VibeVoiceMultipleSpeakersNode"] = VibeVoiceMultipleSpeakersNode + NODE_DISPLAY_NAME_MAPPINGS["VibeVoiceMultipleSpeakersNode"] = "VibeVoice Multiple Speakers" + + # Free memory node + NODE_CLASS_MAPPINGS["VibeVoiceFreeMemoryNode"] = VibeVoiceFreeMemoryNode + NODE_DISPLAY_NAME_MAPPINGS["VibeVoiceFreeMemoryNode"] = "VibeVoice Free Memory" + + # LoRA configuration node + NODE_CLASS_MAPPINGS["VibeVoiceLoRANode"] = VibeVoiceLoRANode + NODE_DISPLAY_NAME_MAPPINGS["VibeVoiceLoRANode"] = "VibeVoice LoRA" + + logger.info("VibeVoice nodes registered successfully") + + except Exception as e: + logger.error(f"Failed to register VibeVoice nodes: {e}") + logger.info("Please ensure transformers>=4.44.0 is installed") +else: + logger.warning("VibeVoice nodes unavailable - check embedded module and dependencies") + +__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', '__version__'] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/__pycache__/__init__.cpython-312.pyc b/VibeVoice-ComfyUI/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..43701c9b4154d372a85e75af6c2857cec5dc0f12 Binary files /dev/null and b/VibeVoice-ComfyUI/__pycache__/__init__.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/examples/Multiple-Speaker.json b/VibeVoice-ComfyUI/examples/Multiple-Speaker.json new file mode 100644 index 0000000000000000000000000000000000000000..e8cd547bc01ca62b92ccd499d75b1d623bcd5886 --- /dev/null +++ b/VibeVoice-ComfyUI/examples/Multiple-Speaker.json @@ -0,0 +1 @@ +{"id":"e5ca15c5-18b5-4d37-8852-795692a14b29","revision":0,"last_node_id":38,"last_link_id":57,"nodes":[{"id":19,"type":"LoadTextFromFileNode","pos":[9.889446258544922,621.1560668945312],"size":[270,58],"flags":{},"order":0,"mode":4,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null}],"outputs":[{"localized_name":"text","name":"text","type":"STRING","links":null}],"properties":{"Node name for S&R":"LoadTextFromFileNode","cnr_id":"VibeVoice-ComfyUI","ver":"5a24489a7b0bf0c406d291dd51e82a085d338d44"},"widgets_values":["No text files found in any directory"],"color":"#323","bgcolor":"#535"},{"id":31,"type":"Note","pos":[379.3583984375,725.9093627929688],"size":[415,88],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[],"title":"Voice Speed Factor","properties":{},"widgets_values":["The voice speed factor influences the original source audio to attempt to achieve a slower or faster final speech. 1.0 is the normal speed. It is recommended not to exceed values ​​between 0.95 and 1.05. The effect is best when you provide a sample audio of at least 20 seconds."],"color":"#432","bgcolor":"#653"},{"id":16,"type":"PreviewAudio","pos":[896.3719482421875,189.1308135986328],"size":[270,88],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":57},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"PreviewAudio"},"widgets_values":[],"color":"#323","bgcolor":"#535"},{"id":20,"type":"Note","pos":[-55.931907653808594,726.6131591796875],"size":[415,88],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[],"title":"Load Text From File","properties":{},"widgets_values":["Use Load Text From File if you want to use a .txt file instead of text-area. You can load .txt files from ComfyUI/input, ComfyUI/output or ComfyUI/temp directories."],"color":"#432","bgcolor":"#653"},{"id":15,"type":"LoadAudio","pos":[-12.263749122619629,190.64144897460938],"size":[270,136],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[55]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"LoadAudio"},"widgets_values":["Voice1.mp3",null,null],"color":"#2a363b","bgcolor":"#3f5159"},{"id":17,"type":"LoadAudio","pos":[-11.774602890014648,403.2247009277344],"size":[270,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[56]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"LoadAudio"},"widgets_values":["Voice2.mp3",null,null],"color":"#2a363b","bgcolor":"#3f5159"},{"id":36,"type":"VibeVoiceMultipleSpeakersNode","pos":[393.1620178222656,189.6568145751953],"size":[400,456],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"speaker1_voice","name":"speaker1_voice","shape":7,"type":"AUDIO","link":55},{"localized_name":"speaker2_voice","name":"speaker2_voice","shape":7,"type":"AUDIO","link":56},{"localized_name":"speaker3_voice","name":"speaker3_voice","shape":7,"type":"AUDIO","link":null},{"localized_name":"speaker4_voice","name":"speaker4_voice","shape":7,"type":"AUDIO","link":null},{"localized_name":"lora","name":"lora","shape":7,"type":"LORA_CONFIG","link":null},{"localized_name":"text","name":"text","type":"STRING","widget":{"name":"text"},"link":null},{"localized_name":"model","name":"model","type":"COMBO","widget":{"name":"model"},"link":null},{"localized_name":"attention_type","name":"attention_type","type":"COMBO","widget":{"name":"attention_type"},"link":null},{"localized_name":"quantize_llm","name":"quantize_llm","type":"COMBO","widget":{"name":"quantize_llm"},"link":null},{"localized_name":"free_memory_after_generate","name":"free_memory_after_generate","type":"BOOLEAN","widget":{"name":"free_memory_after_generate"},"link":null},{"localized_name":"diffusion_steps","name":"diffusion_steps","type":"INT","widget":{"name":"diffusion_steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"cfg_scale","name":"cfg_scale","type":"FLOAT","widget":{"name":"cfg_scale"},"link":null},{"localized_name":"use_sampling","name":"use_sampling","type":"BOOLEAN","widget":{"name":"use_sampling"},"link":null},{"localized_name":"temperature","name":"temperature","shape":7,"type":"FLOAT","widget":{"name":"temperature"},"link":null},{"localized_name":"top_p","name":"top_p","shape":7,"type":"FLOAT","widget":{"name":"top_p"},"link":null},{"localized_name":"voice_speed_factor","name":"voice_speed_factor","shape":7,"type":"FLOAT","widget":{"name":"voice_speed_factor"},"link":null}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[57]}],"properties":{"Node name for S&R":"VibeVoiceMultipleSpeakersNode"},"widgets_values":["[1]: Hello, this is the first speaker.\n[2]: Hi there, I'm the second speaker.\n[1]: Nice to meet you!\n[2]: Nice to meet you too!","VibeVoice-Large","auto","4bit",true,20,42,"fixed",1.3,false,0.95,0.95,1],"color":"#223","bgcolor":"#335"},{"id":37,"type":"Note","pos":[-530.1146850585938,279.4844055175781],"size":[408.66363525390625,236.39089965820312],"flags":{},"order":5,"mode":0,"inputs":[],"outputs":[],"title":"1) Download Models","properties":{},"widgets_values":["You have to manually download the models you would like to use and put them into: ComfyUI/models/vibevoice/\n\nMake a directory for each model and put all the files inside them.\n\nVibeVoice-1.5B model (~ 5.4 GB):\nhttps://huggingface.co/microsoft/VibeVoice-1.5B/tree/main\n\nVibeVoice-Large model (~ 18.7 GB):\nhttps://huggingface.co/aoi-ot/VibeVoice-Large/tree/main\n\nVibeVoice-Large-Q-8bit model (~ 11.6 GB):\nhttps://huggingface.co/FabioSarracino/VibeVoice-Large-Q8/tree/main\n\nVibeVoice-Large-Q-4bit model (~ 6.6 GB):\nhttps://huggingface.co/DevParker/VibeVoice7b-low-vram/tree/main/4bit"],"color":"#432","bgcolor":"#653"},{"id":38,"type":"Note","pos":[-529.8153686523438,579.2252807617188],"size":[407.2561950683594,155.19009399414062],"flags":{},"order":6,"mode":0,"inputs":[],"outputs":[],"title":"2) Download Tokenizer","properties":{},"widgets_values":["You have to manually download the Qwen2.5 Tokenizer files and put them into: ComfyUI/models/vibevoice/tokenizer/\n\nhttps://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n\nRequired files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json (~11MB)\n\nPut the files directly inside tokenizer directory without make another directory inside."],"color":"#432","bgcolor":"#653"}],"links":[[55,15,0,36,0,"AUDIO"],[56,17,0,36,1,"AUDIO"],[57,36,0,16,0,"AUDIO"]],"groups":[{"id":1,"title":"Instructions before use:","bounding":[-553.0167846679688,181.94606018066406,453.3775939941406,595.2697143554688],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"ds":{"scale":0.9090909090909097,"offset":[944.6168885013626,-55.446182500052494]}},"version":0.4} \ No newline at end of file diff --git a/VibeVoice-ComfyUI/examples/Pause-Tag.json b/VibeVoice-ComfyUI/examples/Pause-Tag.json new file mode 100644 index 0000000000000000000000000000000000000000..9a1e70f56b50364cf69f354c0ea3975ec82dc135 --- /dev/null +++ b/VibeVoice-ComfyUI/examples/Pause-Tag.json @@ -0,0 +1 @@ +{"id":"b70cf6f7-8531-4faa-9843-9c963a4ba577","revision":0,"last_node_id":47,"last_link_id":58,"nodes":[{"id":28,"type":"LoadTextFromFileNode","pos":[-51.13530731201172,497.1748352050781],"size":[289.5152282714844,58],"flags":{},"order":0,"mode":4,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null}],"outputs":[{"localized_name":"text","name":"text","type":"STRING","links":null}],"properties":{"Node name for S&R":"LoadTextFromFileNode","cnr_id":"VibeVoice-ComfyUI","ver":"5a24489a7b0bf0c406d291dd51e82a085d338d44"},"widgets_values":["No text files found in any directory"],"color":"#323","bgcolor":"#535"},{"id":38,"type":"Note","pos":[775.2548828125,307.8158874511719],"size":[415,88],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[],"title":"Pause System","properties":{},"widgets_values":["[pause]: add 1 second of silence.\n[pause:{number}] add {number}ms of pause\nWARNING: the pause tag forces the text to be split into chunks. This may worsen the model’s ability to understand the context. The model’s context is represented ONLY by its own chunk."],"color":"#432","bgcolor":"#653"},{"id":15,"type":"LoadAudio","pos":[-52.503074645996094,163.9591064453125],"size":[270,136],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[57]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"LoadAudio"},"widgets_values":["Voice.mp3",null,null],"color":"#2a363b","bgcolor":"#3f5159"},{"id":21,"type":"Note","pos":[-119.67156219482422,637.6148071289062],"size":[415,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Load Text From File","properties":{},"widgets_values":["Use Load Text From File if you want to use a .txt file instead of text-area. You can load .txt files from ComfyUI/input, ComfyUI/output or ComfyUI/temp directories."],"color":"#432","bgcolor":"#653"},{"id":16,"type":"PreviewAudio","pos":[845.1698608398438,163.10276794433594],"size":[270,88],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":58},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"PreviewAudio"},"widgets_values":[],"color":"#323","bgcolor":"#535"},{"id":40,"type":"Note","pos":[325.02294921875,636.903564453125],"size":[415,88],"flags":{},"order":4,"mode":0,"inputs":[],"outputs":[],"title":"Voice Speed Factor","properties":{},"widgets_values":["The voice speed factor influences the original source audio to attempt to achieve a slower or faster final speech. 1.0 is the normal speed. It is recommended not to exceed values ​​between 0.95 and 1.05. The effect is best when you provide a sample audio of at least 20 seconds."],"color":"#432","bgcolor":"#653"},{"id":45,"type":"VibeVoiceSingleSpeakerNode","pos":[327.48126220703125,164.61436462402344],"size":[400,420],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"voice_to_clone","name":"voice_to_clone","shape":7,"type":"AUDIO","link":57},{"localized_name":"lora","name":"lora","shape":7,"type":"LORA_CONFIG","link":null},{"localized_name":"text","name":"text","type":"STRING","widget":{"name":"text"},"link":null},{"localized_name":"model","name":"model","type":"COMBO","widget":{"name":"model"},"link":null},{"localized_name":"attention_type","name":"attention_type","type":"COMBO","widget":{"name":"attention_type"},"link":null},{"localized_name":"quantize_llm","name":"quantize_llm","type":"COMBO","widget":{"name":"quantize_llm"},"link":null},{"localized_name":"free_memory_after_generate","name":"free_memory_after_generate","type":"BOOLEAN","widget":{"name":"free_memory_after_generate"},"link":null},{"localized_name":"diffusion_steps","name":"diffusion_steps","type":"INT","widget":{"name":"diffusion_steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"cfg_scale","name":"cfg_scale","type":"FLOAT","widget":{"name":"cfg_scale"},"link":null},{"localized_name":"use_sampling","name":"use_sampling","type":"BOOLEAN","widget":{"name":"use_sampling"},"link":null},{"localized_name":"temperature","name":"temperature","shape":7,"type":"FLOAT","widget":{"name":"temperature"},"link":null},{"localized_name":"top_p","name":"top_p","shape":7,"type":"FLOAT","widget":{"name":"top_p"},"link":null},{"localized_name":"max_words_per_chunk","name":"max_words_per_chunk","shape":7,"type":"INT","widget":{"name":"max_words_per_chunk"},"link":null},{"localized_name":"voice_speed_factor","name":"voice_speed_factor","shape":7,"type":"FLOAT","widget":{"name":"voice_speed_factor"},"link":null}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[58]}],"properties":{"Node name for S&R":"VibeVoiceSingleSpeakerNode"},"widgets_values":["Hello, this is a test of the VibeVoice text-to-speech system. [pause] Do you like my voice? [pause:500] What's your name?","VibeVoice-1.5B","auto","full precision",true,20,42,"fixed",1.3,false,0.95,0.95,250,1],"color":"#223","bgcolor":"#335"},{"id":46,"type":"Note","pos":[-576.477294921875,222.4726104736328],"size":[408.66363525390625,236.39089965820312],"flags":{},"order":5,"mode":0,"inputs":[],"outputs":[],"title":"1) Download Models","properties":{},"widgets_values":["You have to manually download the models you would like to use and put them into: ComfyUI/models/vibevoice/\n\nMake a directory for each model and put all the files inside them.\n\nVibeVoice-1.5B model (~ 5.4 GB):\nhttps://huggingface.co/microsoft/VibeVoice-1.5B/tree/main\n\nVibeVoice-Large model (~ 18.7 GB):\nhttps://huggingface.co/aoi-ot/VibeVoice-Large/tree/main\n\nVibeVoice-Large-Q-8bit model (~ 11.6 GB):\nhttps://huggingface.co/FabioSarracino/VibeVoice-Large-Q8/tree/main\n\nVibeVoice-Large-Q-4bit model (~ 6.6 GB):\nhttps://huggingface.co/DevParker/VibeVoice7b-low-vram/tree/main/4bit"],"color":"#432","bgcolor":"#653"},{"id":47,"type":"Note","pos":[-576.177978515625,522.212646484375],"size":[407.2561950683594,155.19009399414062],"flags":{},"order":6,"mode":0,"inputs":[],"outputs":[],"title":"2) Download Tokenizer","properties":{},"widgets_values":["You have to manually download the Qwen2.5 Tokenizer files and put them into: ComfyUI/models/vibevoice/tokenizer/\n\nhttps://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n\nRequired files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json (~11MB)\n\nPut the files directly inside tokenizer directory without make another directory inside."],"color":"#432","bgcolor":"#653"}],"links":[[57,15,0,45,0,"AUDIO"],[58,45,0,16,0,"AUDIO"]],"groups":[{"id":2,"title":"Instructions before use:","bounding":[-599.3793334960938,124.93412017822266,453.3775939941406,595.2697143554688],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"ds":{"scale":0.8264462809917354,"offset":[815.9689977237014,-22.084207406969263]}},"version":0.4} \ No newline at end of file diff --git a/VibeVoice-ComfyUI/examples/Single-Speaker.json b/VibeVoice-ComfyUI/examples/Single-Speaker.json new file mode 100644 index 0000000000000000000000000000000000000000..2ce1db40f9bb5a938858b832963d57056c871a35 --- /dev/null +++ b/VibeVoice-ComfyUI/examples/Single-Speaker.json @@ -0,0 +1 @@ +{"id":"c6ef8963-032c-45f6-954f-b5f6b354343b","revision":0,"last_node_id":44,"last_link_id":61,"nodes":[{"id":15,"type":"LoadAudio","pos":[15.256911277770996,126.44892883300781],"size":[270,136],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[60]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"LoadAudio"},"widgets_values":["Voice.mp3",null,null],"color":"#2a363b","bgcolor":"#3f5159"},{"id":21,"type":"Note","pos":[-83.88814544677734,580.3738403320312],"size":[415,88],"flags":{},"order":4,"mode":0,"inputs":[],"outputs":[],"title":"Load Text From File","properties":{},"widgets_values":["Use Load Text From File if you want to use a .txt file instead of text-area. You can load .txt files from ComfyUI/input, ComfyUI/output or ComfyUI/temp directories."],"color":"#432","bgcolor":"#653"},{"id":40,"type":"Note","pos":[377.95758056640625,593.4078979492188],"size":[415,88],"flags":{},"order":5,"mode":0,"inputs":[],"outputs":[],"title":"Voice Speed Factor","properties":{},"widgets_values":["The voice speed factor influences the original source audio to attempt to achieve a slower or faster final speech. 1.0 is the normal speed. It is recommended not to exceed values ​​between 0.95 and 1.05. The effect is best when you provide a sample audio of at least 20 seconds."],"color":"#432","bgcolor":"#653"},{"id":16,"type":"PreviewAudio","pos":[894.1837768554688,126.69258117675781],"size":[270,88],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":61},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"PreviewAudio"},"widgets_values":[],"color":"#323","bgcolor":"#535"},{"id":44,"type":"VibeVoiceSingleSpeakerNode","pos":[388.8460693359375,126.70189666748047],"size":[400,420],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"voice_to_clone","name":"voice_to_clone","shape":7,"type":"AUDIO","link":60},{"localized_name":"lora","name":"lora","shape":7,"type":"LORA_CONFIG","link":null},{"localized_name":"text","name":"text","type":"STRING","widget":{"name":"text"},"link":null},{"localized_name":"model","name":"model","type":"COMBO","widget":{"name":"model"},"link":null},{"localized_name":"attention_type","name":"attention_type","type":"COMBO","widget":{"name":"attention_type"},"link":null},{"localized_name":"quantize_llm","name":"quantize_llm","type":"COMBO","widget":{"name":"quantize_llm"},"link":null},{"localized_name":"free_memory_after_generate","name":"free_memory_after_generate","type":"BOOLEAN","widget":{"name":"free_memory_after_generate"},"link":null},{"localized_name":"diffusion_steps","name":"diffusion_steps","type":"INT","widget":{"name":"diffusion_steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"cfg_scale","name":"cfg_scale","type":"FLOAT","widget":{"name":"cfg_scale"},"link":null},{"localized_name":"use_sampling","name":"use_sampling","type":"BOOLEAN","widget":{"name":"use_sampling"},"link":null},{"localized_name":"temperature","name":"temperature","shape":7,"type":"FLOAT","widget":{"name":"temperature"},"link":null},{"localized_name":"top_p","name":"top_p","shape":7,"type":"FLOAT","widget":{"name":"top_p"},"link":null},{"localized_name":"max_words_per_chunk","name":"max_words_per_chunk","shape":7,"type":"INT","widget":{"name":"max_words_per_chunk"},"link":null},{"localized_name":"voice_speed_factor","name":"voice_speed_factor","shape":7,"type":"FLOAT","widget":{"name":"voice_speed_factor"},"link":null}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[61]}],"properties":{"Node name for S&R":"VibeVoiceSingleSpeakerNode"},"widgets_values":["Hello, this is a test of the VibeVoice text-to-speech system.","VibeVoice-1.5B","auto","full precision",true,20,42,"fixed",1.3,false,0.95,0.95,250,1],"color":"#223","bgcolor":"#335"},{"id":28,"type":"LoadTextFromFileNode","pos":[-11.502296447753906,465.8179626464844],"size":[289.5152282714844,58],"flags":{},"order":0,"mode":4,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null}],"outputs":[{"localized_name":"text","name":"text","type":"STRING","links":null}],"properties":{"Node name for S&R":"LoadTextFromFileNode","cnr_id":"VibeVoice-ComfyUI","ver":"5a24489a7b0bf0c406d291dd51e82a085d338d44"},"widgets_values":["No text files found in any directory"],"color":"#323","bgcolor":"#535"},{"id":22,"type":"Note","pos":[-539.2780151367188,186.78372192382812],"size":[408.66363525390625,236.39089965820312],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[],"title":"1) Download Models","properties":{},"widgets_values":["You have to manually download the models you would like to use and put them into: ComfyUI/models/vibevoice/\n\nMake a directory for each model and put all the files inside them.\n\nVibeVoice-1.5B model (~ 5.4 GB):\nhttps://huggingface.co/microsoft/VibeVoice-1.5B/tree/main\n\nVibeVoice-Large model (~ 18.7 GB):\nhttps://huggingface.co/aoi-ot/VibeVoice-Large/tree/main\n\nVibeVoice-Large-Q-8bit model (~ 11.6 GB):\nhttps://huggingface.co/FabioSarracino/VibeVoice-Large-Q8/tree/main\n\nVibeVoice-Large-Q-4bit model (~ 6.6 GB):\nhttps://huggingface.co/DevParker/VibeVoice7b-low-vram/tree/main/4bit"],"color":"#432","bgcolor":"#653"},{"id":42,"type":"Note","pos":[-538.9786987304688,486.52374267578125],"size":[407.2561950683594,155.19009399414062],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[],"title":"2) Download Tokenizer","properties":{},"widgets_values":["You have to manually download the Qwen2.5 Tokenizer files and put them into: ComfyUI/models/vibevoice/tokenizer/\n\nhttps://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n\nRequired files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json (~11MB)\n\nPut the files directly inside tokenizer directory without make another directory inside."],"color":"#432","bgcolor":"#653"}],"links":[[60,15,0,44,0,"AUDIO"],[61,44,0,16,0,"AUDIO"]],"groups":[{"id":2,"title":"Instructions before use:","bounding":[-562.1800537109375,89.24514770507812,453.3775939941406,595.2697143554688],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"ds":{"scale":0.9090909090909091,"offset":[795.8030854327329,-23.374334793282447]}},"version":0.4} \ No newline at end of file diff --git a/VibeVoice-ComfyUI/examples/VibeVoice-Unload-Memory.json b/VibeVoice-ComfyUI/examples/VibeVoice-Unload-Memory.json new file mode 100644 index 0000000000000000000000000000000000000000..9146b28691ab511dfff7f81770365c070d89ef29 --- /dev/null +++ b/VibeVoice-ComfyUI/examples/VibeVoice-Unload-Memory.json @@ -0,0 +1 @@ +{"id":"fc471b7e-ccef-427f-be3f-29dec93a90ea","revision":0,"last_node_id":45,"last_link_id":56,"nodes":[{"id":34,"type":"VibeVoiceFreeMemoryNode","pos":[913.2552490234375,127.35599517822266],"size":[189.03964233398438,26],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":56}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[42]}],"properties":{"Node name for S&R":"VibeVoiceFreeMemoryNode","cnr_id":"VibeVoice-ComfyUI","ver":"5a24489a7b0bf0c406d291dd51e82a085d338d44"},"widgets_values":[],"color":"#322","bgcolor":"#533"},{"id":16,"type":"PreviewAudio","pos":[1273.2957763671875,127.3007583618164],"size":[270,88],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":42},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"PreviewAudio"},"widgets_values":[],"color":"#323","bgcolor":"#535"},{"id":35,"type":"Note","pos":[809.6192016601562,208.98324584960938],"size":[432.1000061035156,126.30000305175781],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[],"title":"Free Memory Node","properties":{},"widgets_values":["The VibeVoice Free Memory node releases memory as soon as it receives the audio input (acting as a passthrough for the audio itself). In this specific use case, however, it’s redundant, since it would be enough to enable the “free_memory_after_generate” parameter of the previous node. The ideal use case is, for example, when you have a loop generating multiple audio clips, and only after the final generation you pass the last audio and free the memory."],"color":"#432","bgcolor":"#653"},{"id":28,"type":"LoadTextFromFileNode","pos":[-30.95530128479004,453.30511474609375],"size":[289.5152282714844,58],"flags":{},"order":2,"mode":4,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null}],"outputs":[{"localized_name":"text","name":"text","type":"STRING","links":null}],"properties":{"Node name for S&R":"LoadTextFromFileNode","cnr_id":"VibeVoice-ComfyUI","ver":"5a24489a7b0bf0c406d291dd51e82a085d338d44"},"widgets_values":["No text files found in any directory"],"color":"#323","bgcolor":"#535"},{"id":40,"type":"Note","pos":[367.98895263671875,597.8056640625],"size":[415,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Voice Speed Factor","properties":{},"widgets_values":["The voice speed factor influences the original source audio to attempt to achieve a slower or faster final speech. 1.0 is the normal speed. It is recommended not to exceed values ​​between 0.95 and 1.05. The effect is best when you provide a sample audio of at least 20 seconds."],"color":"#432","bgcolor":"#653"},{"id":15,"type":"LoadAudio","pos":[-21.549091339111328,127.7799301147461],"size":[270,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[55]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.49","Node name for S&R":"LoadAudio"},"widgets_values":["Voice.mp3",null,null],"color":"#2a363b","bgcolor":"#3f5159"},{"id":43,"type":"VibeVoiceSingleSpeakerNode","pos":[373.596435546875,128.40489196777344],"size":[400,420],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"voice_to_clone","name":"voice_to_clone","shape":7,"type":"AUDIO","link":55},{"localized_name":"lora","name":"lora","shape":7,"type":"LORA_CONFIG","link":null},{"localized_name":"text","name":"text","type":"STRING","widget":{"name":"text"},"link":null},{"localized_name":"model","name":"model","type":"COMBO","widget":{"name":"model"},"link":null},{"localized_name":"attention_type","name":"attention_type","type":"COMBO","widget":{"name":"attention_type"},"link":null},{"localized_name":"quantize_llm","name":"quantize_llm","type":"COMBO","widget":{"name":"quantize_llm"},"link":null},{"localized_name":"free_memory_after_generate","name":"free_memory_after_generate","type":"BOOLEAN","widget":{"name":"free_memory_after_generate"},"link":null},{"localized_name":"diffusion_steps","name":"diffusion_steps","type":"INT","widget":{"name":"diffusion_steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"cfg_scale","name":"cfg_scale","type":"FLOAT","widget":{"name":"cfg_scale"},"link":null},{"localized_name":"use_sampling","name":"use_sampling","type":"BOOLEAN","widget":{"name":"use_sampling"},"link":null},{"localized_name":"temperature","name":"temperature","shape":7,"type":"FLOAT","widget":{"name":"temperature"},"link":null},{"localized_name":"top_p","name":"top_p","shape":7,"type":"FLOAT","widget":{"name":"top_p"},"link":null},{"localized_name":"max_words_per_chunk","name":"max_words_per_chunk","shape":7,"type":"INT","widget":{"name":"max_words_per_chunk"},"link":null},{"localized_name":"voice_speed_factor","name":"voice_speed_factor","shape":7,"type":"FLOAT","widget":{"name":"voice_speed_factor"},"link":null}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[56]}],"properties":{"Node name for S&R":"VibeVoiceSingleSpeakerNode"},"widgets_values":["Hello, this is a test of the VibeVoice text-to-speech system.","VibeVoice-1.5B","auto","full precision",true,20,42,"fixed",1.3,false,0.95,0.95,250,1],"color":"#223","bgcolor":"#335"},{"id":44,"type":"Note","pos":[-546.2021484375,184.94338989257812],"size":[408.66363525390625,236.39089965820312],"flags":{},"order":5,"mode":0,"inputs":[],"outputs":[],"title":"1) Download Models","properties":{},"widgets_values":["You have to manually download the models you would like to use and put them into: ComfyUI/models/vibevoice/\n\nMake a directory for each model and put all the files inside them.\n\nVibeVoice-1.5B model (~ 5.4 GB):\nhttps://huggingface.co/microsoft/VibeVoice-1.5B/tree/main\n\nVibeVoice-Large model (~ 18.7 GB):\nhttps://huggingface.co/aoi-ot/VibeVoice-Large/tree/main\n\nVibeVoice-Large-Q-8bit model (~ 11.6 GB):\nhttps://huggingface.co/FabioSarracino/VibeVoice-Large-Q8/tree/main\n\nVibeVoice-Large-Q-4bit model (~ 6.6 GB):\nhttps://huggingface.co/DevParker/VibeVoice7b-low-vram/tree/main/4bit"],"color":"#432","bgcolor":"#653"},{"id":45,"type":"Note","pos":[-545.90283203125,484.68328857421875],"size":[407.2561950683594,155.19009399414062],"flags":{},"order":6,"mode":0,"inputs":[],"outputs":[],"title":"2) Download Tokenizer","properties":{},"widgets_values":["You have to manually download the Qwen2.5 Tokenizer files and put them into: ComfyUI/models/vibevoice/tokenizer/\n\nhttps://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n\nRequired files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json (~11MB)\n\nPut the files directly inside tokenizer directory without make another directory inside."],"color":"#432","bgcolor":"#653"},{"id":21,"type":"Note","pos":[-84.54156494140625,599.46435546875],"size":[415,88],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Load Text From File","properties":{},"widgets_values":["Use Load Text From File if you want to use a .txt file instead of text-area. You can load .txt files from ComfyUI/input, ComfyUI/output or ComfyUI/temp directories."],"color":"#432","bgcolor":"#653"}],"links":[[42,34,0,16,0,"AUDIO"],[55,15,0,43,0,"AUDIO"],[56,43,0,34,0,"AUDIO"]],"groups":[{"id":2,"title":"Instructions before use:","bounding":[-569.1041870117188,87.40498352050781,453.3775939941406,595.2697143554688],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"ds":{"scale":0.9090909090909091,"offset":[570.2036733851843,-33.504933709055805]}},"version":0.4} \ No newline at end of file diff --git a/VibeVoice-ComfyUI/examples/podcast_note.txt b/VibeVoice-ComfyUI/examples/podcast_note.txt new file mode 100644 index 0000000000000000000000000000000000000000..1fb78377ca74d38bf0c2f18ee664d8c5bcae181e --- /dev/null +++ b/VibeVoice-ComfyUI/examples/podcast_note.txt @@ -0,0 +1 @@ +नमस्कार दोस्तों, मैं हूँ आपकी होस्ट नेहा, और आप सुन रहे हैं "कहानियों की दुनिया"। [pause:800] आज मैं आपको एक बहुत ही दिलचस्प कहानी सुनाने वाली हूँ। [pause:800] यह कहानी है एक छोटी सी लड़की की, जिसका नाम था मीरा। [pause:800] मीरा एक छोटे से गाँव में रहती थी, जो चारों तरफ से पहाड़ों से घिरा हुआ था। [pause:800] उस गाँव में एक पुरानी कहावत थी कि पहाड़ की सबसे ऊँची चोटी पर एक जादुई पेड़ है। [pause:800] कहते थे कि जो भी उस पेड़ के नीचे अपनी सबसे सच्ची इच्छा बोलेगा, वो पूरी हो जाएगी। [pause:800] लेकिन वहाँ तक पहुँचना इतना आसान नहीं था। [pause:800] रास्ते में घना जंगल था, अंधेरी गुफाएँ थीं, और एक ऐसी नदी थी जो उल्टी दिशा में बहती थी। [pause:800] गाँव के लोग कहते थे कि वहाँ जाना पागलपन है। [pause:800] लेकिन मीरा कोई साधारण लड़की नहीं थी। [pause:800] उसकी माँ बहुत बीमार थीं, और कोई भी दवाई काम नहीं कर रही थी। [pause:800] तो एक रात मीरा ने फैसला किया कि वो उस जादुई पेड़ तक जाएगी। [pause:800] अगली सुबह सूरज निकलने से पहले ही मीरा अपना छोटा सा झोला उठाकर चल पड़ी। [pause:800] जंगल में अंधेरा इतना गहरा था कि अपना हाथ भी नहीं दिखता था। [pause:800] तभी अचानक एक छोटी सी चिड़िया आई, जिसके पंख चाँदनी की तरह चमक रहे थे। [pause:800] वो चिड़िया मीरा के आगे-आगे उड़ने लगी, जैसे रास्ता दिखा रही हो। [pause:800] मीरा ने उसका साथ पकड़ा और आगे बढ़ती गई। [pause:800] फिर आई वो उल्टी बहने वाली नदी। [pause:800] मीरा ने देखा कि नदी में पत्थर एक खास पैटर्न में रखे हैं। [pause:800] उसने हिम्मत करके एक-एक पत्थर पर पैर रखते हुए नदी पार कर ली। [pause:800] और आखिरकार वो पहुँच गई उस चोटी पर। [pause:800] वहाँ सच में एक बहुत बड़ा पेड़ था, जिसकी पत्तियाँ सोने जैसी चमक रही थीं। [pause:800] मीरा ने आँखें बंद कीं और बोली, "बस मेरी माँ ठीक हो जाएँ।" [pause:800] एक पत्ती टूटकर मीरा की हथेली पर गिरी और रोशनी में बदल गई। [pause:800] जब मीरा घर लौटी, तो उसकी माँ बिस्तर पर बैठी मुस्कुरा रही थीं। [pause:800] उनका बुखार उतर चुका था, चेहरे पर रंग आ गया था। [pause:800] गाँव वाले इसे चमत्कार कहते हैं। [pause:800] लेकिन मैं कहती हूँ कि यह चमत्कार नहीं, यह एक बेटी की हिम्मत और प्यार की ताकत थी। [pause:800] क्योंकि जब दिल से कोई चाहता है ना, तो रास्ते खुद बन जाते हैं। [pause:800] तो दोस्तों, आज की कहानी से यही सीख मिलती है कि हिम्मत कभी मत हारो। [pause:800] अगर कहानी अच्छी लगी हो तो शेयर ज़रूर करना। [pause:800] मिलते हैं अगली बार एक और नई कहानी के साथ। [pause:800] आपकी होस्ट नेहा, अलविदा! \ No newline at end of file diff --git a/VibeVoice-ComfyUI/node_list.json b/VibeVoice-ComfyUI/node_list.json new file mode 100644 index 0000000000000000000000000000000000000000..ca48957ed1adced72060cc57c923c8538f78a2cd --- /dev/null +++ b/VibeVoice-ComfyUI/node_list.json @@ -0,0 +1,7 @@ +{ + "VibeVoice Load Text From File": "Load .txt from ComfyUI input/output/temp", + "VibeVoice Single Speaker": "Single-speaker TTS with optional voice cloning", + "VibeVoice Multiple Speakers": "Multi-speaker TTS ([1]..[4]) with optional clones", + "VibeVoice Free Memory": "Frees loaded VibeVoice models; passthrough audio", + "VibeVoice LoRA": "Configure LoRA adapters for fine-tuned VibeVoice models" +} \ No newline at end of file diff --git a/VibeVoice-ComfyUI/nodes/__init__.py b/VibeVoice-ComfyUI/nodes/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..37bea57a461ac091275f3c42711697d570df08b0 --- /dev/null +++ b/VibeVoice-ComfyUI/nodes/__init__.py @@ -0,0 +1,17 @@ +# Created by Fabio Sarracino +# Nodes module for VibeVoiceWrapper +""" +This module contains all the ComfyUI nodes for VibeVoice integration. +""" + +from .load_text_node import LoadTextFromFileNode +from .single_speaker_node import VibeVoiceSingleSpeakerNode +from .multi_speaker_node import VibeVoiceMultipleSpeakersNode +from .free_memory_node import VibeVoiceFreeMemoryNode + +__all__ = [ + 'LoadTextFromFileNode', + 'VibeVoiceSingleSpeakerNode', + 'VibeVoiceMultipleSpeakersNode', + 'VibeVoiceFreeMemoryNode' +] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/nodes/__pycache__/__init__.cpython-312.pyc b/VibeVoice-ComfyUI/nodes/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc52ad1acaa96b568890633541cd2be3146d8c93 Binary files /dev/null and b/VibeVoice-ComfyUI/nodes/__pycache__/__init__.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/nodes/__pycache__/base_vibevoice.cpython-312.pyc b/VibeVoice-ComfyUI/nodes/__pycache__/base_vibevoice.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba17b495c60f03fb0d27cdbf531bd49b8e06a61b Binary files /dev/null and b/VibeVoice-ComfyUI/nodes/__pycache__/base_vibevoice.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/nodes/__pycache__/free_memory_node.cpython-312.pyc b/VibeVoice-ComfyUI/nodes/__pycache__/free_memory_node.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c19b986a5291cbc81df9ecc46871a46f543a3f02 Binary files /dev/null and b/VibeVoice-ComfyUI/nodes/__pycache__/free_memory_node.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/nodes/__pycache__/load_text_node.cpython-312.pyc b/VibeVoice-ComfyUI/nodes/__pycache__/load_text_node.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9eb42d3712f489631acf130cf5b09c74f15c7eba Binary files /dev/null and b/VibeVoice-ComfyUI/nodes/__pycache__/load_text_node.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/nodes/__pycache__/lora_node.cpython-312.pyc b/VibeVoice-ComfyUI/nodes/__pycache__/lora_node.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7207e32d7535de34b20dd44a735f0b10873843fe Binary files /dev/null and b/VibeVoice-ComfyUI/nodes/__pycache__/lora_node.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/nodes/__pycache__/multi_speaker_node.cpython-312.pyc b/VibeVoice-ComfyUI/nodes/__pycache__/multi_speaker_node.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25ae8ccaac830fca2ae11654337797ac8ec065bf Binary files /dev/null and b/VibeVoice-ComfyUI/nodes/__pycache__/multi_speaker_node.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/nodes/__pycache__/single_speaker_node.cpython-312.pyc b/VibeVoice-ComfyUI/nodes/__pycache__/single_speaker_node.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..678bf0e65e3f0aaf72419baa783bdd9b4977218d Binary files /dev/null and b/VibeVoice-ComfyUI/nodes/__pycache__/single_speaker_node.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/nodes/base_vibevoice.py b/VibeVoice-ComfyUI/nodes/base_vibevoice.py new file mode 100644 index 0000000000000000000000000000000000000000..0f8bc0c893031534a54ce3c794a2128dae65b3b4 --- /dev/null +++ b/VibeVoice-ComfyUI/nodes/base_vibevoice.py @@ -0,0 +1,1728 @@ +# Created by Fabio Sarracino +# Base class for VibeVoice nodes with common functionality + +import logging +import os +import tempfile +import torch +import numpy as np +import re +import gc +import json +from typing import List, Optional, Tuple, Any, Dict + +# Setup logging +logger = logging.getLogger("VibeVoice") + +# Import for interruption support +try: + import execution + INTERRUPTION_SUPPORT = True +except ImportError: + INTERRUPTION_SUPPORT = False + logger.warning("Interruption support not available") + +# Check for SageAttention availability +try: + from sageattention import sageattn + SAGE_AVAILABLE = True + logger.info("SageAttention available for acceleration") +except ImportError: + SAGE_AVAILABLE = False + logger.debug("SageAttention not available - install with: pip install sageattention") + +def get_optimal_device(): + """Get the best available device (cuda, mps, or cpu)""" + if torch.cuda.is_available(): + return "cuda" + elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + return "mps" + else: + return "cpu" + +def get_device_map(): + """Get device map for model loading""" + device = get_optimal_device() + # Note: device_map "auto" might work better for MPS in some cases + return device if device != "mps" else "mps" + +# Cache for model scanning to avoid repeated scans +_model_cache = { + "models": None, + "last_scan_time": 0, + "cache_duration": 5, # Cache for 5 seconds + "first_load_logged": False # Track if we've logged the initial scan +} + +def get_available_models() -> List[Tuple[str, str]]: + """Scan models/vibevoice/ directory and return available models + + Returns: + List of tuples (display_name, folder_path) + """ + import time + + # Check if we have a valid cache + current_time = time.time() + if (_model_cache["models"] is not None and + current_time - _model_cache["last_scan_time"] < _model_cache["cache_duration"]): + # Return cached results + return _model_cache["models"] + + try: + import folder_paths + models_dir = folder_paths.get_folder_paths("checkpoints")[0] + vibevoice_dir = os.path.join(os.path.dirname(models_dir), "vibevoice") + + if not os.path.exists(vibevoice_dir): + os.makedirs(vibevoice_dir, exist_ok=True) + logger.info(f"Created vibevoice models directory: {vibevoice_dir}") + _model_cache["models"] = [] + _model_cache["last_scan_time"] = current_time + return [] + + # First, collect all valid model folders + valid_folders = [] + logger.debug(f"Scanning vibevoice directory: {vibevoice_dir}") + for folder in os.listdir(vibevoice_dir): + folder_path = os.path.join(vibevoice_dir, folder) + + # Skip hidden folders, loras, and non-directories + if folder.startswith(".") or folder == "loras" or not os.path.isdir(folder_path): + logger.debug(f"Skipping: {folder}") + continue + + logger.debug(f"Checking folder: {folder}") + # Check if it's a valid model folder + if is_valid_model_folder(folder_path): + valid_folders.append(folder) + else: + logger.debug(f"Folder {folder} is not a valid model folder") + + # Now transform folder names with duplicate detection + models = [] + for folder in valid_folders: + display_name = transform_folder_name(folder, valid_folders) + models.append((folder, display_name)) + logger.debug(f"Found model: {display_name} in folder: {folder}") + + # Sort by display name for consistent ordering + models.sort(key=lambda x: x[1]) + + # Only log on first scan to avoid spam + if not _model_cache["first_load_logged"]: + if not models: + logger.warning("No valid models found in vibevoice directory") + logger.info(f"Please download models to: {vibevoice_dir}") + else: + # Single summary message instead of individual logs + logger.info(f"Found {len(models)} VibeVoice model(s) available") + _model_cache["first_load_logged"] = True + + # Cache the results + _model_cache["models"] = models + _model_cache["last_scan_time"] = current_time + + return models + + except Exception as e: + logger.error(f"Error scanning models directory: {e}") + # Cache empty result on error to avoid repeated failures + _model_cache["models"] = [] + _model_cache["last_scan_time"] = current_time + return [] + +def extract_model_info(folder: str) -> Tuple[str, Optional[str]]: + """Extract model name and author from folder name + + Args: + folder: Folder name + + Returns: + Tuple of (model_name, author_name) + + Examples: + models--microsoft--VibeVoice-Large -> ('VibeVoice-Large', 'microsoft') + models--aoi-ot--VibeVoice-Large -> ('VibeVoice-Large', 'aoi-ot') + VibeVoice-1.5B -> ('VibeVoice-1.5B', None) + """ + if "--" in folder: + # HuggingFace cache format: models--author--model + parts = folder.split("--") + if len(parts) >= 3: + author = parts[1] + model = parts[-1] + return model, author + elif len(parts) == 2: + return parts[-1], None + return folder, None + +def transform_folder_name(folder: str, all_folders: List[str]) -> str: + """Transform folder name for display, adding author if there are duplicates + + Args: + folder: Current folder name + all_folders: List of all folder names to check for duplicates + + Returns: + Display name with author in parentheses if needed + """ + model_name, author = extract_model_info(folder) + + # Check if there are other folders with the same model name + has_duplicate = False + for other_folder in all_folders: + if other_folder != folder: + other_model_name, _ = extract_model_info(other_folder) + if other_model_name == model_name: + has_duplicate = True + break + + # Add author in parentheses if there are duplicates and author is known + if has_duplicate and author: + return f"{model_name} ({author})" + + return model_name + +def check_folder_has_model_files(folder_path: str) -> bool: + """Check if a folder directly contains model files (not recursively) + + Args: + folder_path: Path to check + + Returns: + True if folder contains config.json and model files + """ + if not os.path.isdir(folder_path): + return False + + has_config = os.path.exists(os.path.join(folder_path, "config.json")) + if not has_config: + return False + + # Check for various model file formats + files = os.listdir(folder_path) + has_model = ( + "pytorch_model.bin" in files or + "model.safetensors" in files or + "pytorch_model.bin.index.json" in files or + "model.safetensors.index.json" in files or + any(f.startswith("pytorch_model-") and f.endswith(".bin") for f in files) or + any(f.startswith("model-") and f.endswith(".safetensors") for f in files) + ) + + return has_model + +def is_valid_model_folder(folder_path: str, max_depth: int = 4, current_depth: int = 0) -> bool: + """Recursively check if a folder contains a valid VibeVoice model + + Args: + folder_path: Path to the folder to check + max_depth: Maximum recursion depth (default 3) + current_depth: Current recursion depth + + Returns: + True if folder or any subfolder contains valid model files + """ + if current_depth >= max_depth: + return False + + # Check if current folder has model files + if check_folder_has_model_files(folder_path): + return True + + # Recursively check subfolders + try: + for item in os.listdir(folder_path): + # Skip hidden folders and specific folders we want to ignore + if item.startswith(".") or item in ["loras", "__pycache__"]: + continue + + item_path = os.path.join(folder_path, item) + if os.path.isdir(item_path): + # Recursively check subfolder + if is_valid_model_folder(item_path, max_depth, current_depth + 1): + return True + except (PermissionError, OSError): + pass + + return False + +def find_model_files_path_recursive(folder_path: str, max_depth: int = 4, current_depth: int = 0) -> Optional[str]: + """Recursively find the path containing model files + + Args: + folder_path: Path to search from + max_depth: Maximum recursion depth + current_depth: Current recursion depth + + Returns: + Path to the directory containing model files, or None + """ + if current_depth >= max_depth: + return None + + # Check if current folder has model files + if check_folder_has_model_files(folder_path): + return folder_path + + # Recursively check subfolders + try: + for item in os.listdir(folder_path): + # Skip hidden folders and specific folders we want to ignore + if item.startswith(".") or item in ["loras", "__pycache__"]: + continue + + item_path = os.path.join(folder_path, item) + if os.path.isdir(item_path): + # Recursively check subfolder + result = find_model_files_path_recursive(item_path, max_depth, current_depth + 1) + if result: + return result + except (PermissionError, OSError): + pass + + return None + +def find_model_files_path(model_folder: str) -> Optional[str]: + """Find the actual path containing model files + + Args: + model_folder: Name of the folder in vibevoice directory + + Returns: + Path to the directory containing model files, or None + """ + try: + import folder_paths + models_dir = folder_paths.get_folder_paths("checkpoints")[0] + vibevoice_dir = os.path.join(os.path.dirname(models_dir), "vibevoice") + base_path = os.path.join(vibevoice_dir, model_folder) + + # Use recursive search to find model files + result = find_model_files_path_recursive(base_path) + + if result: + logger.info(f"Found model files at: {result}") + else: + logger.warning(f"No valid model files found for: {model_folder}") + + return result + + except Exception as e: + logger.error(f"Error finding model files: {e}") + return None + +def find_qwen_tokenizer_path(comfyui_models_dir: str) -> Optional[str]: + """Find Qwen tokenizer using priority system + + Priority: + 1. ComfyUI/models/vibevoice/tokenizer/ + 2. ComfyUI/models/vibevoice/models--Qwen--Qwen2.5-1.5B/ + 3. HuggingFace cache (if exists) + + Returns: + Path to tokenizer directory or None + """ + # Priority 1: Check tokenizer folder + tokenizer_dir = os.path.join(comfyui_models_dir, "tokenizer") + if os.path.exists(tokenizer_dir): + # Check if it contains tokenizer files + required_files = ["tokenizer_config.json", "vocab.json", "merges.txt"] + if all(os.path.exists(os.path.join(tokenizer_dir, f)) for f in required_files): + logger.info(f"Found Qwen tokenizer in: {tokenizer_dir}") + return tokenizer_dir + + # Priority 2: Check models--Qwen--Qwen2.5-1.5B folder + qwen_model_dir = os.path.join(comfyui_models_dir, "models--Qwen--Qwen2.5-1.5B") + if os.path.exists(qwen_model_dir): + # Check snapshots folder + snapshots_dir = os.path.join(qwen_model_dir, "snapshots") + if os.path.exists(snapshots_dir): + for snapshot in os.listdir(snapshots_dir): + snapshot_path = os.path.join(snapshots_dir, snapshot) + if os.path.isdir(snapshot_path): + # Check if it contains tokenizer files + if os.path.exists(os.path.join(snapshot_path, "tokenizer_config.json")): + logger.info(f"Found Qwen tokenizer in model cache: {snapshot_path}") + return snapshot_path + + # Priority 3: Check HuggingFace cache + hf_cache_paths = [ + os.path.expanduser("~/.cache/huggingface/hub"), + os.path.join(os.environ.get("HF_HOME", ""), "hub") if os.environ.get("HF_HOME") else None, + ] + + for cache_path in hf_cache_paths: + if cache_path and os.path.exists(cache_path): + qwen_cache = os.path.join(cache_path, "models--Qwen--Qwen2.5-1.5B") + if os.path.exists(qwen_cache): + snapshots_dir = os.path.join(qwen_cache, "snapshots") + if os.path.exists(snapshots_dir): + for snapshot in os.listdir(snapshots_dir): + snapshot_path = os.path.join(snapshots_dir, snapshot) + if os.path.isdir(snapshot_path): + if os.path.exists(os.path.join(snapshot_path, "tokenizer_config.json")): + logger.info(f"Found Qwen tokenizer in HF cache: {snapshot_path}") + return snapshot_path + + return None + +def detect_model_quantization(model_path: str) -> Optional[str]: + """Detect if model is quantized from config files + + Args: + model_path: Path to the model directory + + Returns: + '4bit', '8bit', or None + """ + try: + # Check for quantization_config.json first + quant_config_path = os.path.join(model_path, "quantization_config.json") + if os.path.exists(quant_config_path): + with open(quant_config_path, 'r') as f: + quant_config = json.load(f) + if quant_config.get("load_in_4bit"): + return "4bit" + if quant_config.get("load_in_8bit"): + return "8bit" + + # Check main config.json + config_path = os.path.join(model_path, "config.json") + if os.path.exists(config_path): + with open(config_path, 'r') as f: + config = json.load(f) + if "quantization_config" in config: + if config["quantization_config"].get("load_in_4bit"): + return "4bit" + if config["quantization_config"].get("load_in_8bit"): + return "8bit" + if config["quantization_config"].get("bits") == 4: + return "4bit" + + except Exception as e: + logger.debug(f"Could not detect quantization: {e}") + + return None + +class BaseVibeVoiceNode: + """Base class for VibeVoice nodes containing common functionality""" + + def __init__(self): + self.model = None + self.processor = None + self.current_model_folder = None + self.current_attention_type = None + self.current_quantize_llm = "full precision" + self.current_lora_path = None + # LoRA component flags (overridable by node instances) + self.use_llm_lora = True + self.use_diffusion_head_lora = True + self.use_acoustic_connector_lora = True + self.use_semantic_connector_lora = True + + def free_memory(self): + """Free model and processor from memory""" + try: + if self.model is not None: + del self.model + self.model = None + + if self.processor is not None: + del self.processor + self.processor = None + + self.current_model_folder = None + self.current_quantize_llm = "full precision" + + # Force garbage collection and clear CUDA cache if available + import gc + gc.collect() + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.synchronize() + + logger.info("Model and processor memory freed successfully") + + except Exception as e: + logger.error(f"Error freeing memory: {e}") + + def _check_dependencies(self): + """Check if VibeVoice is available and import it with fallback installation""" + try: + import sys + import os + + # Add vvembed to path + current_dir = os.path.dirname(os.path.abspath(__file__)) + parent_dir = os.path.dirname(current_dir) + vvembed_path = os.path.join(parent_dir, 'vvembed') + + if vvembed_path not in sys.path: + sys.path.insert(0, vvembed_path) + + # Import from embedded version + from modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference + + logger.info(f"Using embedded VibeVoice from {vvembed_path}") + return None, VibeVoiceForConditionalGenerationInference + + except ImportError as e: + logger.error(f"Embedded VibeVoice import failed: {e}") + + # Try fallback to installed version if available + try: + import vibevoice + from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference + logger.warning("Falling back to system-installed VibeVoice") + return vibevoice, VibeVoiceForConditionalGenerationInference + except ImportError: + pass + + raise Exception( + "VibeVoice embedded module import failed. Please ensure the vvembed folder exists " + "and transformers>=4.51.3 is installed." + ) + + def _apply_lora(self, lora_path: str): + """Apply LoRA adapters to the model""" + try: + logger.info(f"Starting LoRA application from path: {lora_path}") + + # Check component flags + use_llm = getattr(self, 'use_llm_lora', True) + use_diffusion = getattr(self, 'use_diffusion_head_lora', True) + use_acoustic = getattr(self, 'use_acoustic_connector_lora', True) + use_semantic = getattr(self, 'use_semantic_connector_lora', True) + + logger.info(f"LoRA component flags - LLM: {use_llm}, Diffusion: {use_diffusion}, Acoustic: {use_acoustic}, Semantic: {use_semantic}") + + if not any([use_llm, use_diffusion, use_acoustic, use_semantic]): + logger.info("All LoRA components disabled, skipping LoRA application") + return + + # Apply LLM LoRA adapter if requested + if use_llm: + # Check if adapter files exist + adapter_model_path = os.path.join(lora_path, "adapter_model.safetensors") + adapter_bin_path = os.path.join(lora_path, "adapter_model.bin") + adapter_config = os.path.join(lora_path, "adapter_config.json") + + has_adapter = os.path.exists(adapter_model_path) or os.path.exists(adapter_bin_path) + + if has_adapter and os.path.exists(adapter_config): + try: + from peft import PeftModel + base_lm = getattr(self.model.model, 'language_model', None) + if base_lm is not None: + logger.info(f"Applying LLM LoRA adapter from: {lora_path}") + lora_wrapped = PeftModel.from_pretrained(base_lm, lora_path, is_trainable=False) + device = next(self.model.parameters()).device + dtype = next(self.model.parameters()).dtype + lora_wrapped = lora_wrapped.to(device=device, dtype=dtype) + self.model.model.language_model = lora_wrapped + logger.info("LLM LoRA adapter successfully applied") + except ImportError: + logger.warning("PEFT library not available for LLM LoRA") + except Exception as e: + logger.warning(f"Failed to apply LLM LoRA: {e}") + else: + logger.info(f"No LLM LoRA adapter files found in {lora_path}, skipping LLM LoRA") + + # Helper function to load state dict into module + def _load_state_dict_into(module, folder): + if module is None: + logger.warning(f"Module is None, cannot load state dict from {folder}") + return False + if not os.path.isdir(folder): + logger.warning(f"Folder does not exist: {folder}") + return False + + try: + # Try safetensors first + safetensor_path = os.path.join(folder, "model.safetensors") + if os.path.exists(safetensor_path): + try: + import safetensors.torch as st + logger.info(f"Loading safetensor from: {safetensor_path}") + state_dict = st.load_file(safetensor_path) + logger.info(f"Loaded state dict with {len(state_dict)} keys") + + # Get device and dtype from module + device = next(module.parameters()).device + dtype = next(module.parameters()).dtype + logger.info(f"Module device: {device}, dtype: {dtype}") + + # Convert state dict to correct device and dtype + for key in state_dict: + state_dict[key] = state_dict[key].to(device=device, dtype=dtype) + + # Load with strict=False to allow partial loading + missing_keys, unexpected_keys = module.load_state_dict(state_dict, strict=False) + + if missing_keys: + logger.warning(f"Missing keys when loading state dict ({len(missing_keys)} total): {missing_keys[:5]}..." if len(missing_keys) > 5 else f"Missing keys: {missing_keys}") + if unexpected_keys: + logger.warning(f"Unexpected keys when loading state dict ({len(unexpected_keys)} total): {unexpected_keys[:5]}..." if len(unexpected_keys) > 5 else f"Unexpected keys: {unexpected_keys}") + + # Log success even with missing keys if most were loaded + total_keys = len(state_dict) + if missing_keys: + logger.info(f"Loaded {total_keys} keys from LoRA, {len(missing_keys)} keys not found in model") + + logger.info("Successfully loaded state dict into module") + return True + except Exception as e: + logger.warning(f"Failed to load safetensors: {e}") + import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") + + # Fallback to PyTorch format + pytorch_path = os.path.join(folder, "pytorch_model.bin") + if os.path.exists(pytorch_path): + logger.info(f"Loading pytorch model from: {pytorch_path}") + state_dict = torch.load(pytorch_path, map_location="cpu") + logger.info(f"Loaded state dict with {len(state_dict)} keys") + + # Get device and dtype from module + device = next(module.parameters()).device + dtype = next(module.parameters()).dtype + + # Convert state dict to correct device and dtype + for key in state_dict: + state_dict[key] = state_dict[key].to(device=device, dtype=dtype) + + missing_keys, unexpected_keys = module.load_state_dict(state_dict, strict=False) + + if missing_keys: + logger.warning(f"Missing keys: {missing_keys[:5]}..." if len(missing_keys) > 5 else f"Missing keys: {missing_keys}") + if unexpected_keys: + logger.warning(f"Unexpected keys: {unexpected_keys[:5]}..." if len(unexpected_keys) > 5 else f"Unexpected keys: {unexpected_keys}") + + logger.info("Successfully loaded pytorch model into module") + return True + else: + logger.warning(f"No model file found in {folder}") + logger.warning(f"Looked for: {safetensor_path} and {pytorch_path}") + + except Exception as e: + logger.error(f"Failed to load state dict from {folder}: {e}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + return False + + # Load diffusion head if requested + if use_diffusion: + diffusion_path = os.path.join(lora_path, "diffusion_head") + if os.path.exists(diffusion_path): + logger.info(f"Found diffusion_head directory at: {diffusion_path}") + + # The diffusion head is called 'prediction_head' in VibeVoice + module = getattr(self.model.model, 'prediction_head', None) + if module: + logger.info("Found prediction_head module in model") + + # Check model compatibility by looking at dimensions + skip_loading = False + try: + # Get hidden size from the module + if hasattr(module, 'cond_proj') and hasattr(module.cond_proj, 'weight'): + model_hidden_size = module.cond_proj.weight.shape[0] + logger.info(f"Current model prediction_head hidden size: {model_hidden_size}") + + # Check LoRA dimensions + safetensor_path = os.path.join(diffusion_path, "model.safetensors") + if os.path.exists(safetensor_path): + import safetensors.torch as st + lora_state = st.load_file(safetensor_path) + if 'cond_proj.weight' in lora_state: + lora_hidden_size = lora_state['cond_proj.weight'].shape[0] + logger.info(f"LoRA diffusion head hidden size: {lora_hidden_size}") + + if model_hidden_size != lora_hidden_size: + skip_loading = True + if lora_hidden_size == 3584: + logger.error("="*60) + logger.error("LoRA MODEL MISMATCH!") + logger.error(f"This LoRA was trained on VibeVoice-Large (hidden_size=3584)") + if model_hidden_size == 1536: + logger.error(f"You are using VibeVoice-1.5B (hidden_size=1536)") + logger.error("Please switch to 'VibeVoice-Large' model to use this LoRA") + else: + logger.error(f"Your model has hidden_size={model_hidden_size}") + logger.error("Please use VibeVoice-Large (non-quantized) model") + logger.error("="*60) + logger.error("Skipping LoRA loading due to incompatible model") + elif lora_hidden_size == 1536: + logger.error("="*60) + logger.error("LoRA MODEL MISMATCH!") + logger.error(f"This LoRA was trained on VibeVoice-1.5B (hidden_size=1536)") + logger.error(f"You are using a model with hidden_size={model_hidden_size}") + logger.error("Please switch to 'VibeVoice-1.5B' model to use this LoRA") + logger.error("="*60) + logger.error("Skipping LoRA loading due to incompatible model") + except Exception as e: + logger.debug(f"Could not check model compatibility: {e}") + + # Only attempt to load if compatible + if not skip_loading: + if _load_state_dict_into(module, diffusion_path): + logger.info("Diffusion head LoRA loaded successfully into prediction_head") + else: + logger.warning("Failed to load diffusion head LoRA") + else: + logger.info("Diffusion head LoRA loading skipped due to model mismatch") + else: + logger.warning("Model does not have prediction_head attribute") + # Debug: list available attributes + attrs = [a for a in dir(self.model.model) if not a.startswith('_')] + logger.debug(f"Available model.model attributes: {attrs[:15]}...") + else: + logger.info(f"No diffusion_head directory found at: {diffusion_path}") + + # Load acoustic connector if requested + if use_acoustic: + acoustic_path = os.path.join(lora_path, "acoustic_connector") + if os.path.exists(acoustic_path): + module = getattr(self.model.model, 'acoustic_connector', None) + if module and _load_state_dict_into(module, acoustic_path): + logger.info("Acoustic connector LoRA loaded") + + # Load semantic connector if requested + if use_semantic: + semantic_path = os.path.join(lora_path, "semantic_connector") + if os.path.exists(semantic_path): + module = getattr(self.model.model, 'semantic_connector', None) + if module and _load_state_dict_into(module, semantic_path): + logger.info("Semantic connector LoRA loaded") + + + # Log summary of what was loaded + logger.info("LoRA application completed") + + except Exception as e: + logger.error(f"Error applying LoRA: {e}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + # Don't fail the entire load, just log the error + + def _verify_quantization(self, expected_mode: str): + """Verify that quantization was actually applied correctly""" + try: + quantized_layers = [] + fp_layers = [] + + for name, module in self.model.named_modules(): + if isinstance(module, torch.nn.Linear): + module_type = type(module).__name__ + + if 'Linear8bitLt' in module_type or '8bit' in module_type.lower(): + quantized_layers.append((name, '8bit')) + elif 'Linear4bit' in module_type or '4bit' in module_type.lower(): + quantized_layers.append((name, '4bit')) + else: + fp_layers.append(name) + + # Concise summary + total_linear = len(quantized_layers) + len(fp_layers) + + if len(quantized_layers) > 0: + pct = 100 * len(quantized_layers) / total_linear + logger.info(f"✅ {expected_mode} quantization: {len(quantized_layers)}/{total_linear} layers ({pct:.1f}%)") + else: + logger.warning(f"⚠️ No {expected_mode} quantization detected") + + except Exception as e: + logger.debug(f"Could not verify quantization: {e}") + + def _apply_sage_attention(self): + """Apply SageAttention to the loaded model by monkey-patching attention layers""" + try: + from sageattention import sageattn + import torch.nn.functional as F + + # Counter for patched layers + patched_count = 0 + + def patch_attention_forward(module): + """Recursively patch attention layers to use SageAttention""" + nonlocal patched_count + + # Check if this module has scaled_dot_product_attention + if hasattr(module, 'forward'): + original_forward = module.forward + + # Create wrapper that replaces F.scaled_dot_product_attention with sageattn + def sage_forward(*args, **kwargs): + # Temporarily replace F.scaled_dot_product_attention + original_sdpa = F.scaled_dot_product_attention + + def sage_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None, **kwargs): + """Wrapper that converts sdpa calls to sageattn""" + # Log any unexpected parameters for debugging + if kwargs: + unexpected_params = list(kwargs.keys()) + logger.debug(f"SageAttention: Ignoring unsupported parameters: {unexpected_params}") + + try: + # SageAttention expects tensors in specific format + # Transformers typically use (batch, heads, seq_len, head_dim) + + # Check tensor dimensions to determine layout + if query.dim() == 4: + # 4D tensor: (batch, heads, seq, dim) + batch_size = query.shape[0] + num_heads = query.shape[1] + seq_len_q = query.shape[2] + seq_len_k = key.shape[2] + head_dim = query.shape[3] + + # Reshape to (batch*heads, seq, dim) for HND layout + query_reshaped = query.reshape(batch_size * num_heads, seq_len_q, head_dim) + key_reshaped = key.reshape(batch_size * num_heads, seq_len_k, head_dim) + value_reshaped = value.reshape(batch_size * num_heads, seq_len_k, head_dim) + + # Call sageattn with HND layout + output = sageattn( + query_reshaped, key_reshaped, value_reshaped, + is_causal=is_causal, + tensor_layout="HND" # Heads*batch, seqN, Dim + ) + + # Output should be (batch*heads, seq_len_q, head_dim) + # Reshape back to (batch, heads, seq, dim) + if output.dim() == 3: + output = output.reshape(batch_size, num_heads, seq_len_q, head_dim) + + return output + else: + # For 3D tensors, assume they're already in HND format + output = sageattn( + query, key, value, + is_causal=is_causal, + tensor_layout="HND" + ) + return output + + except Exception as e: + # If SageAttention fails, fall back to original implementation + logger.debug(f"SageAttention failed, using original: {e}") + # Call with proper arguments - scale is a keyword argument in PyTorch 2.0+ + # Pass through any additional kwargs that the original sdpa might support + if scale is not None: + return original_sdpa(query, key, value, attn_mask=attn_mask, + dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) + else: + return original_sdpa(query, key, value, attn_mask=attn_mask, + dropout_p=dropout_p, is_causal=is_causal, **kwargs) + + # Replace the function + F.scaled_dot_product_attention = sage_sdpa + + try: + # Call original forward with patched attention + result = original_forward(*args, **kwargs) + finally: + # Restore original function + F.scaled_dot_product_attention = original_sdpa + + return result + + # Check if this module likely uses attention + # Look for common attention module names + module_name = module.__class__.__name__.lower() + if any(name in module_name for name in ['attention', 'attn', 'multihead']): + module.forward = sage_forward + patched_count += 1 + + # Recursively patch child modules + for child in module.children(): + patch_attention_forward(child) + + # Apply patching to the entire model + patch_attention_forward(self.model) + + logger.info(f"Patched {patched_count} attention layers with SageAttention") + + if patched_count == 0: + logger.warning("No attention layers found to patch - SageAttention may not be applied") + + except Exception as e: + logger.error(f"Failed to apply SageAttention: {e}") + logger.warning("Continuing with standard attention implementation") + + def load_model(self, model_name: str, model_folder: str, attention_type: str = "auto", quantize_llm: str = "full precision", lora_path: str = None): + """Load VibeVoice model with specified attention implementation and optional LoRA + + Args: + model_name: The display name of the model (e.g., "VibeVoice-Large") + model_folder: The folder name in models/vibevoice/ containing the model + attention_type: The attention implementation to use + quantize_llm: LLM quantization mode ("full precision", "8bit", or "4bit") + lora_path: Optional path to LoRA adapter directory + """ + # Check if we need to reload model due to attention type, quantization, or LoRA change + current_attention = getattr(self, 'current_attention_type', None) + current_quantize_llm = getattr(self, 'current_quantize_llm', 'full precision') + current_lora = getattr(self, 'current_lora_path', None) + lora_changed = (current_lora or "") != (lora_path or "") + quantize_changed = current_quantize_llm != quantize_llm + + if (self.model is None or + getattr(self, 'current_model_folder', None) != model_folder or + current_attention != attention_type or + quantize_changed or + lora_changed): + + # Free existing model before loading new one (important for attention type, quantization, or LoRA changes) + if self.model is not None and (current_attention != attention_type or quantize_changed or getattr(self, 'current_model_folder', None) != model_folder or lora_changed): + logger.info(f"Freeing existing model before loading with new settings (attention: {current_attention} -> {attention_type}, quantize: {current_quantize_llm} -> {quantize_llm}, LoRA: {current_lora} -> {lora_path})") + self.free_memory() + + try: + vibevoice, VibeVoiceInferenceModel = self._check_dependencies() + + # Set ComfyUI models directory + import folder_paths + models_dir = folder_paths.get_folder_paths("checkpoints")[0] + comfyui_models_dir = os.path.join(os.path.dirname(models_dir), "vibevoice") + os.makedirs(comfyui_models_dir, exist_ok=True) + + # Import time for timing + import time + start_time = time.time() + + # Suppress verbose logs + import transformers + import warnings + transformers.logging.set_verbosity_error() + warnings.filterwarnings("ignore", category=UserWarning) + + # Get the actual model path using our discovery function + model_full_path = os.path.join(comfyui_models_dir, model_folder) + + # Find where the actual model files are + model_files_path = find_model_files_path(model_folder) + + if not model_files_path: + raise Exception(f"No valid model files found in {model_full_path}. Please ensure the model is properly downloaded.") + + logger.info(f"Found model files at: {model_files_path}") + + # Check if model files are in a 4bit subfolder + use_4bit_subfolder = False + actual_model_path = model_files_path + if model_files_path.endswith(os.sep + "4bit"): + # If files are in 4bit subfolder, use parent path and set subfolder + actual_model_path = os.path.dirname(model_files_path) + use_4bit_subfolder = True + logger.info(f"Model uses 4bit subfolder structure") + + # Detect if model is quantized + quantization = detect_model_quantization(model_files_path) + if quantization: + logger.info(f"Detected {quantization} quantization") + + # Check if this is a quantized model + is_quantized_4bit = quantization == "4bit" + is_quantized_8bit = quantization == "8bit" + is_quantized = is_quantized_4bit or is_quantized_8bit + + # Prepare attention implementation kwargs + model_kwargs = { + "cache_dir": comfyui_models_dir, + "trust_remote_code": True, + "torch_dtype": torch.bfloat16, + "device_map": get_device_map(), + } + + # Handle quantized model loading + if is_quantized_4bit or is_quantized_8bit: + # Check if CUDA is available (required for quantization) + if not torch.cuda.is_available(): + raise Exception("Quantized models require a CUDA GPU. Please use standard models on CPU/MPS.") + + # Try to import bitsandbytes + try: + from transformers import BitsAndBytesConfig + + if is_quantized_4bit: + logger.info("Loading 4-bit quantized model with bitsandbytes...") + # Configure 4-bit quantization + bnb_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.bfloat16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4' + ) + if use_4bit_subfolder: + model_kwargs["subfolder"] = "4bit" + logger.info("Using subfolder='4bit' for loading") + else: # 8-bit + logger.info("Loading 8-bit quantized model with bitsandbytes...") + # Configure 8-bit quantization + bnb_config = BitsAndBytesConfig( + load_in_8bit=True, + bnb_8bit_compute_dtype=torch.bfloat16 + ) + + model_kwargs["quantization_config"] = bnb_config + model_kwargs["device_map"] = "cuda" # Force CUDA for quantized models + + except ImportError: + raise Exception( + "Quantized models require 'bitsandbytes' library.\n" + "Please install it with: pip install bitsandbytes\n" + "Or use the standard VibeVoice models instead." + ) + + # Handle LLM-only 8-bit quantization (for non-quantized models) - EXPERIMENTAL + elif quantize_llm == "8bit" and not is_quantized: + # Check if CUDA is available (required for quantization) + if not torch.cuda.is_available(): + raise Exception("LLM quantization requires a CUDA GPU. Please use 'full precision' on CPU/MPS.") + + # Try to import bitsandbytes + try: + from transformers import BitsAndBytesConfig + + logger.info("Quantizing LLM component to 8-bit...") + # Configure 8-bit quantization for LLM only + # CRITICAL: Must skip all audio-related components to prevent noise + bnb_config = BitsAndBytesConfig( + load_in_8bit=True, + bnb_8bit_compute_dtype=torch.bfloat16, + # Skip ALL audio-critical components (same as 4bit + more conservative) + llm_int8_skip_modules=[ + "lm_head", # Output projection + "prediction_head", # Diffusion head - CRITICAL for audio quality + "acoustic_connector", # Audio->LLM projection - CRITICAL + "semantic_connector", # Semantic->LLM projection - CRITICAL + "acoustic_tokenizer", # VAE encoder/decoder for audio + "semantic_tokenizer", # VAE encoder for semantics + ], + # Ultra-conservative outlier threshold (lower = more fp16 processing) + # Default is 6.0, but audio/diffusion models need 3.0-4.0 for stability + llm_int8_threshold=3.0, + # Disable fp16 weights (use int8 storage) + llm_int8_has_fp16_weight=False, + ) + + model_kwargs["quantization_config"] = bnb_config + model_kwargs["device_map"] = "auto" + + # Flag for post-load verification + model_kwargs["_quantization_mode"] = "8bit" + + except ImportError: + raise Exception( + "LLM quantization requires 'bitsandbytes' library.\n" + "Please install it with: pip install bitsandbytes\n" + "Or use 'full precision' mode instead." + ) + + # Handle LLM-only 4-bit quantization (for non-quantized models) + elif quantize_llm == "4bit" and not is_quantized: + # Check if CUDA is available (required for quantization) + if not torch.cuda.is_available(): + raise Exception("LLM quantization requires a CUDA GPU. Please use 'full precision' on CPU/MPS.") + + # Try to import bitsandbytes + try: + from transformers import BitsAndBytesConfig + + logger.info("Quantizing LLM component to 4-bit...") + # Configure 4-bit quantization for LLM only + # Note: lm_head must be skipped to avoid bitsandbytes assertion errors + bnb_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.bfloat16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4', + # Skip lm_head and non-LLM components to avoid errors + llm_int8_skip_modules=["lm_head", "prediction_head", "acoustic_connector", "semantic_connector", "diffusion_head"] + ) + + model_kwargs["quantization_config"] = bnb_config + model_kwargs["device_map"] = "auto" + logger.info("LLM will be quantized to 4-bit, diffusion head and connectors remain at full precision") + + # Flag for post-load verification + model_kwargs["_quantization_mode"] = "4bit" + + except ImportError: + raise Exception( + "LLM quantization requires 'bitsandbytes' library.\n" + "Please install it with: pip install bitsandbytes\n" + "Or use 'full precision' mode instead." + ) + + # Set attention implementation based on user selection + use_sage_attention = False + if attention_type == "sage": + # SageAttention requires special handling - can't be set via attn_implementation + if not SAGE_AVAILABLE: + logger.warning("SageAttention not installed, falling back to sdpa") + logger.warning("Install with: pip install sageattention") + model_kwargs["attn_implementation"] = "sdpa" + elif not torch.cuda.is_available(): + logger.warning("SageAttention requires CUDA GPU, falling back to sdpa") + model_kwargs["attn_implementation"] = "sdpa" + else: + # Don't set attn_implementation for sage, will apply after loading + use_sage_attention = True + logger.info("Will apply SageAttention after model loading") + elif attention_type != "auto": + model_kwargs["attn_implementation"] = attention_type + logger.info(f"Using {attention_type} attention implementation") + else: + # Auto mode - let transformers decide the best implementation + logger.info("Using auto attention implementation selection") + + # Load the model from local path only + model_kwargs["local_files_only"] = True + + # Extract quantization mode flag before loading (it's not a model parameter) + quant_mode = model_kwargs.pop("_quantization_mode", None) + + try: + # Use the correct path (parent if 4bit subfolder is used) + logger.info(f"Loading model from: {actual_model_path}") + if is_quantized: + logger.info(f"Loading {quantization} quantized model...") + if use_4bit_subfolder: + logger.info(f"Using parent path with subfolder='4bit'") + + self.model = VibeVoiceInferenceModel.from_pretrained( + actual_model_path, + **model_kwargs + ) + except Exception as e: + logger.error(f"Failed to load model from {model_files_path}: {e}") + raise Exception( + f"Failed to load model from {model_files_path}.\n" + f"Please ensure the model files are complete and properly downloaded.\n" + f"Required files: config.json, pytorch_model.bin or model safetensors\n" + f"Error: {str(e)}" + ) + + elapsed = time.time() - start_time + logger.info(f"Model loaded in {elapsed:.2f} seconds") + + # Verify quantization if requested (quant_mode was extracted earlier) + if quant_mode: + self._verify_quantization(quant_mode) + + # Verify model was loaded + if self.model is None: + raise Exception("Model failed to load - model is None after loading") + + # Load processor with proper error handling + from processor.vibevoice_processor import VibeVoiceProcessor + + logger.info("Loading VibeVoice processor...") + processor_kwargs = { + "trust_remote_code": True, + "cache_dir": comfyui_models_dir, + "local_files_only": True + } + + # Add subfolder if needed + if use_4bit_subfolder: + processor_kwargs["subfolder"] = "4bit" + + # Pre-check for Qwen tokenizer - REQUIRED + tokenizer_path = find_qwen_tokenizer_path(comfyui_models_dir) + if not tokenizer_path: + # Tokenizer is required - fail early with clear instructions + logger.error("="*60) + logger.error("QWEN TOKENIZER NOT FOUND!") + logger.error("The VibeVoice processor requires the Qwen2.5-1.5B tokenizer.") + logger.error("") + logger.error("To fix this, please download the tokenizer:") + logger.error("1. Download from: https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main") + logger.error(" Required files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json") + logger.error("2. Place files in ONE of these locations (in order of priority):") + logger.error(f" - {os.path.join(comfyui_models_dir, 'tokenizer')}/ (RECOMMENDED)") + logger.error(f" - {os.path.join(comfyui_models_dir, 'models--Qwen--Qwen2.5-1.5B')}/snapshots/[hash]/") + logger.error("3. Restart ComfyUI and try again") + logger.error("="*60) + raise Exception( + "Qwen tokenizer not found. Please download it manually.\n" + "Download from: https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n" + "Required files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json\n" + f"Place files in: {os.path.join(comfyui_models_dir, 'tokenizer')}/" + ) + + # Validate that all required tokenizer files exist + required_files = ["tokenizer_config.json", "vocab.json", "merges.txt"] + missing_files = [] + for file_name in required_files: + file_path = os.path.join(tokenizer_path, file_name) + if not os.path.exists(file_path): + missing_files.append(file_name) + + if missing_files: + logger.error("="*60) + logger.error(f"TOKENIZER IS INCOMPLETE!") + logger.error(f"Tokenizer folder found at: {tokenizer_path}") + logger.error(f"But missing required files: {', '.join(missing_files)}") + logger.error("") + logger.error("Please download ALL required files from:") + logger.error("https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main") + logger.error("Required files:") + logger.error(" - tokenizer_config.json") + logger.error(" - vocab.json") + logger.error(" - merges.txt") + logger.error(" - tokenizer.json (optional but recommended)") + logger.error("="*60) + raise Exception( + f"Tokenizer is incomplete. Missing files: {', '.join(missing_files)}\n" + "Please download ALL required files from: https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n" + "Required files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json\n" + f"Place them in: {tokenizer_path}/" + ) + + logger.info(f"Found complete tokenizer at: {tokenizer_path}") + # Override the language model path to use local tokenizer + processor_kwargs["language_model_pretrained_name"] = tokenizer_path + # Remove cache_dir to avoid HuggingFace cache interference + processor_kwargs.pop('cache_dir', None) + + try: + # Load processor from same path as model + self.processor = VibeVoiceProcessor.from_pretrained( + actual_model_path, + **processor_kwargs + ) + except Exception as proc_error: + logger.warning(f"Failed to load processor from {model_files_path}: {proc_error}") + + # Check if error is about missing Qwen tokenizer + if ("Qwen" in str(proc_error) or "tokenizer" in str(proc_error).lower()): + logger.info("Processor needs Qwen tokenizer. Searching for tokenizer...") + + # Try to find tokenizer using priority system + tokenizer_path = find_qwen_tokenizer_path(comfyui_models_dir) + + if tokenizer_path: + logger.info(f"Found tokenizer at: {tokenizer_path}") + # Try to load processor with tokenizer path hint + try: + from transformers import AutoTokenizer + # Load tokenizer from the found path + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_path, + trust_remote_code=True, + local_files_only=True + ) + logger.info("Qwen tokenizer loaded successfully from local path") + # Store for later use if needed + self._temp_tokenizer = tokenizer + except Exception as tok_error: + logger.warning(f"Failed to load tokenizer from {tokenizer_path}: {tok_error}") + else: + logger.error("="*60) + logger.error("QWEN TOKENIZER NOT FOUND!") + logger.error("The VibeVoice processor requires the Qwen2.5-1.5B tokenizer.") + logger.error("") + logger.error("To fix this, please download the tokenizer:") + logger.error("1. Download from: https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main") + logger.error(" Required files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json") + logger.error("2. Place files in ONE of these locations:") + logger.error(f" - {os.path.join(comfyui_models_dir, 'tokenizer')}/") + logger.error(f" - {os.path.join(comfyui_models_dir, 'models--Qwen--Qwen2.5-1.5B')}/snapshots/[hash]/") + logger.error("3. Restart ComfyUI and try again") + logger.error("="*60) + raise Exception( + "Qwen tokenizer not found. Please download it manually.\n" + "Download from: https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n" + "Required files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json\n" + f"Place tokenizer files in: {os.path.join(comfyui_models_dir, 'tokenizer')}/" + ) + + logger.info("Attempting to load processor with fallback method...") + + # Fallback: try loading without subfolder + try: + if "subfolder" in processor_kwargs: + del processor_kwargs["subfolder"] + self.processor = VibeVoiceProcessor.from_pretrained( + model_files_path, + **processor_kwargs + ) + except Exception as fallback_error: + logger.error(f"Processor loading failed completely: {fallback_error}") + # Check if it's still about Qwen tokenizer + if "Qwen" in str(fallback_error): + tokenizer_path = find_qwen_tokenizer_path(comfyui_models_dir) + if not tokenizer_path: + raise Exception( + f"Failed to load VibeVoice processor: Missing Qwen tokenizer.\n" + f"Download from: https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/main\n" + f"Required files: tokenizer_config.json, vocab.json, merges.txt, tokenizer.json\n" + f"Place files in: {os.path.join(comfyui_models_dir, 'tokenizer')}/" + ) + + raise Exception( + f"Failed to load VibeVoice processor. Error: {fallback_error}\n" + f"Please ensure transformers>=4.51.3 is installed." + ) + + # Move to appropriate device (skip for quantized models as they use device_map) + # Skip device movement for both pre-quantized models and LLM-quantized models + is_llm_quantized = quantize_llm != "full precision" + if not is_quantized and not is_llm_quantized: + device = get_optimal_device() + if device == "cuda": + self.model = self.model.cuda() + elif device == "mps": + self.model = self.model.to("mps") + else: + logger.info("Quantized model already mapped to device via device_map") + + # Apply SageAttention if requested and available + if use_sage_attention and SAGE_AVAILABLE: + self._apply_sage_attention() + logger.info("SageAttention successfully applied to model") + + # Apply LoRA if provided and path exists + if lora_path and os.path.isdir(lora_path): + self._apply_lora(lora_path) + + self.current_model_folder = model_folder + self.current_attention_type = attention_type + self.current_quantize_llm = quantize_llm + self.current_lora_path = lora_path + + except Exception as e: + logger.error(f"Failed to load VibeVoice model: {str(e)}") + raise Exception(f"Model loading failed: {str(e)}") + + def _create_synthetic_voice_sample(self, speaker_idx: int) -> np.ndarray: + """Create synthetic voice sample for a specific speaker""" + sample_rate = 24000 + duration = 1.0 + samples = int(sample_rate * duration) + + t = np.linspace(0, duration, samples, False) + + # Create realistic voice-like characteristics for each speaker + # Use different base frequencies for different speaker types + base_frequencies = [120, 180, 140, 200] # Mix of male/female-like frequencies + base_freq = base_frequencies[speaker_idx % len(base_frequencies)] + + # Create vowel-like formants (like "ah" sound) - unique per speaker + formant1 = 800 + speaker_idx * 100 # First formant + formant2 = 1200 + speaker_idx * 150 # Second formant + + # Generate more voice-like waveform + voice_sample = ( + # Fundamental with harmonics (voice-like) + 0.6 * np.sin(2 * np.pi * base_freq * t) + + 0.25 * np.sin(2 * np.pi * base_freq * 2 * t) + + 0.15 * np.sin(2 * np.pi * base_freq * 3 * t) + + + # Formant resonances (vowel-like characteristics) + 0.1 * np.sin(2 * np.pi * formant1 * t) * np.exp(-t * 2) + + 0.05 * np.sin(2 * np.pi * formant2 * t) * np.exp(-t * 3) + + + # Natural breath noise (reduced) + 0.02 * np.random.normal(0, 1, len(t)) + ) + + # Add natural envelope (like human speech pattern) + # Quick attack, slower decay with slight vibrato (unique per speaker) + vibrato_freq = 4 + speaker_idx * 0.3 # Slightly different vibrato per speaker + envelope = (np.exp(-t * 0.3) * (1 + 0.1 * np.sin(2 * np.pi * vibrato_freq * t))) + voice_sample *= envelope * 0.08 # Lower volume + + return voice_sample.astype(np.float32) + + def _adjust_voice_speed(self, audio_np: np.ndarray, speed_factor: float, sample_rate: int = 24000) -> np.ndarray: + """Adjust voice speed using time-stretching without changing pitch significantly + + Args: + audio_np: Input audio array + speed_factor: Speed adjustment (0.75 = 25% slower, 1.25 = 25% faster) + sample_rate: Sample rate of the audio + + Returns: + Speed-adjusted audio array + """ + if speed_factor == 1.0: + return audio_np # No change needed + + # Calculate new length + original_length = len(audio_np) + target_length = int(original_length / speed_factor) + + # Use linear interpolation for time-stretching + # This is a simple approach that works reasonably well for small speed changes + original_indices = np.arange(original_length) + target_indices = np.linspace(0, original_length - 1, target_length) + + # Interpolate the audio to the new length + adjusted_audio = np.interp(target_indices, original_indices, audio_np) + + logger.info(f"Adjusted voice speed by factor {speed_factor:.2f} ({original_length} -> {target_length} samples)") + + return adjusted_audio.astype(np.float32) + + def _prepare_audio_from_comfyui(self, voice_audio, target_sample_rate: int = 24000, speed_factor: float = 1.0) -> Optional[np.ndarray]: + """Prepare audio from ComfyUI format to numpy array""" + if voice_audio is None: + return None + + # Extract waveform from ComfyUI audio format + if isinstance(voice_audio, dict) and "waveform" in voice_audio: + waveform = voice_audio["waveform"] + input_sample_rate = voice_audio.get("sample_rate", target_sample_rate) + + # Convert to numpy (handling BFloat16 tensors) + if isinstance(waveform, torch.Tensor): + # Convert to float32 first as numpy doesn't support BFloat16 + audio_np = waveform.cpu().float().numpy() + else: + audio_np = np.array(waveform) + + # Handle different audio shapes + if audio_np.ndim == 3: # (batch, channels, samples) + audio_np = audio_np[0, 0, :] # Take first batch, first channel + elif audio_np.ndim == 2: # (channels, samples) + audio_np = audio_np[0, :] # Take first channel + # If 1D, leave as is + + # Resample if needed + if input_sample_rate != target_sample_rate: + target_length = int(len(audio_np) * target_sample_rate / input_sample_rate) + audio_np = np.interp(np.linspace(0, len(audio_np), target_length), + np.arange(len(audio_np)), audio_np) + + # Ensure audio is in correct range [-1, 1] + audio_max = np.abs(audio_np).max() + if audio_max > 0: + audio_np = audio_np / max(audio_max, 1.0) # Normalize + + # Apply speed adjustment if requested + if speed_factor != 1.0: + audio_np = self._adjust_voice_speed(audio_np, speed_factor, target_sample_rate) + speed_percent = int((speed_factor - 1.0) * 100) + if speed_percent > 0: + logger.info(f"Applied voice speed adjustment: +{speed_percent}% faster") + else: + logger.info(f"Applied voice speed adjustment: {speed_percent}% slower") + + return audio_np.astype(np.float32) + + return None + + def _split_text_into_chunks(self, text: str, max_words: int = 250) -> List[str]: + """Split long text into manageable chunks at sentence boundaries + + Args: + text: The text to split + max_words: Maximum words per chunk (default 250 for safety) + + Returns: + List of text chunks + """ + import re + + # Split into sentences (handling common abbreviations) + # This regex tries to split on sentence endings while avoiding common abbreviations + sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])' + sentences = re.split(sentence_pattern, text) + + # If regex split didn't work well, fall back to simple split + if len(sentences) == 1 and len(text.split()) > max_words: + # Fall back to splitting on any period followed by space + sentences = text.replace('. ', '.|').split('|') + sentences = [s.strip() for s in sentences if s.strip()] + + chunks = [] + current_chunk = [] + current_word_count = 0 + + for sentence in sentences: + sentence = sentence.strip() + if not sentence: + continue + + sentence_words = sentence.split() + sentence_word_count = len(sentence_words) + + # If single sentence is too long, split it further + if sentence_word_count > max_words: + # Split long sentence at commas or semicolons + sub_parts = re.split(r'[,;]', sentence) + for part in sub_parts: + part = part.strip() + if not part: + continue + part_words = part.split() + part_word_count = len(part_words) + + if current_word_count + part_word_count > max_words and current_chunk: + # Save current chunk + chunks.append(' '.join(current_chunk)) + current_chunk = [part] + current_word_count = part_word_count + else: + current_chunk.append(part) + current_word_count += part_word_count + else: + # Check if adding this sentence would exceed the limit + if current_word_count + sentence_word_count > max_words and current_chunk: + # Save current chunk and start new one + chunks.append(' '.join(current_chunk)) + current_chunk = [sentence] + current_word_count = sentence_word_count + else: + # Add sentence to current chunk + current_chunk.append(sentence) + current_word_count += sentence_word_count + + # Add remaining chunk + if current_chunk: + chunks.append(' '.join(current_chunk)) + + # If no chunks were created, return the original text + if not chunks: + chunks = [text] + + logger.info(f"Split text into {len(chunks)} chunks (max {max_words} words each)") + for i, chunk in enumerate(chunks): + word_count = len(chunk.split()) + logger.debug(f"Chunk {i+1}: {word_count} words") + + return chunks + + def _parse_pause_keywords(self, text: str) -> List[Tuple[str, Any]]: + """Parse [pause] and [pause:ms] keywords from text + + Args: + text: Text potentially containing pause keywords + + Returns: + List of tuples: ('text', str) or ('pause', duration_ms) + """ + segments = [] + # Pattern matches [pause] or [pause:1500] where 1500 is milliseconds + pattern = r'\[pause(?::(\d+))?\]' + + last_end = 0 + for match in re.finditer(pattern, text): + # Add text segment before pause (if any) + if match.start() > last_end: + text_segment = text[last_end:match.start()].strip() + if text_segment: # Only add non-empty text segments + segments.append(('text', text_segment)) + + # Add pause segment with duration (default 1000ms = 1 second) + duration_ms = int(match.group(1)) if match.group(1) else 1000 + segments.append(('pause', duration_ms)) + last_end = match.end() + + # Add remaining text after last pause (if any) + if last_end < len(text): + remaining_text = text[last_end:].strip() + if remaining_text: + segments.append(('text', remaining_text)) + + # If no pauses found, return original text as single segment + if not segments: + segments.append(('text', text)) + + logger.debug(f"Parsed text into {len(segments)} segments (including pauses)") + return segments + + def _generate_silence(self, duration_ms: int, sample_rate: int = 24000) -> dict: + """Generate silence audio tensor for specified duration + + Args: + duration_ms: Duration of silence in milliseconds + sample_rate: Sample rate (default 24000 Hz for VibeVoice) + + Returns: + Audio dict with silence waveform + """ + # Calculate number of samples for the duration + num_samples = int(sample_rate * duration_ms / 1000.0) + + # Create silence tensor with shape (1, 1, num_samples) to match audio format + silence_waveform = torch.zeros(1, 1, num_samples, dtype=torch.float32) + + logger.info(f"Generated {duration_ms}ms silence ({num_samples} samples)") + + return { + "waveform": silence_waveform, + "sample_rate": sample_rate + } + + def _format_text_for_vibevoice(self, text: str, speakers: list) -> str: + """Format text with speaker information for VibeVoice using correct format""" + # Remove any newlines from the text to prevent parsing issues + # The processor splits by newline and expects each line to have "Speaker N:" format + text = text.replace('\n', ' ').replace('\r', ' ') + # Clean up multiple spaces + text = ' '.join(text.split()) + + # VibeVoice expects format: "Speaker 1: text" not "Name: text" + if len(speakers) == 1: + return f"Speaker 1: {text}" + else: + # Check if text already has proper Speaker N: format + if re.match(r'^\s*Speaker\s+\d+\s*:', text, re.IGNORECASE): + return text + # If text has name format, convert to Speaker N format + elif any(f"{speaker}:" in text for speaker in speakers): + formatted_text = text + for i, speaker in enumerate(speakers): + formatted_text = formatted_text.replace(f"{speaker}:", f"Speaker {i+1}:") + return formatted_text + else: + # Plain text, assign to first speaker + return f"Speaker 1: {text}" + + def _generate_with_vibevoice(self, formatted_text: str, voice_samples: List[np.ndarray], + cfg_scale: float, seed: int, diffusion_steps: int, use_sampling: bool, + temperature: float = 0.95, top_p: float = 0.95, llm_lora_strength: float = 1.0) -> dict: + """Generate audio using VibeVoice model""" + try: + # Ensure model and processor are loaded + if self.model is None or self.processor is None: + raise Exception("Model or processor not loaded") + + # Set seeds for reproducibility + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) # For multi-GPU + + # Also set numpy seed for any numpy operations + np.random.seed(seed) + + # Set diffusion steps + self.model.set_ddpm_inference_steps(diffusion_steps) + logger.info(f"Starting audio generation with {diffusion_steps} diffusion steps...") + + # Check for interruption before starting generation + if INTERRUPTION_SUPPORT: + try: + import comfy.model_management as mm + + # Check if we're being interrupted right now + # The interrupt flag is reset by ComfyUI before each node execution + # So we only check model_management's throw_exception_if_processing_interrupted + # which is the proper way to check for interruption + mm.throw_exception_if_processing_interrupted() + + except ImportError: + # If comfy.model_management is not available, skip this check + pass + + # Prepare inputs using processor + inputs = self.processor( + [formatted_text], # Wrap text in list + voice_samples=[voice_samples], # Provide voice samples for reference + return_tensors="pt", + return_attention_mask=True + ) + + # Move to device + device = next(self.model.parameters()).device + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()} + + # Estimate tokens for user information (not used as limit) + text_length = len(formatted_text.split()) + estimated_tokens = int(text_length * 2.5) # More accurate estimate for display + + # Log generation start with explanation + logger.info(f"Generating audio with {diffusion_steps} diffusion steps...") + logger.info(f"Note: Progress bar shows max possible tokens, not actual needed (~{estimated_tokens} estimated)") + logger.info("The generation will stop automatically when audio is complete") + + # Create stop check function for interruption support + stop_check_fn = None + if INTERRUPTION_SUPPORT: + def check_comfyui_interrupt(): + """Check if ComfyUI has requested interruption""" + try: + if hasattr(execution, 'PromptExecutor') and hasattr(execution.PromptExecutor, 'interrupted'): + interrupted = execution.PromptExecutor.interrupted + if interrupted: + logger.info("Generation interrupted by user via stop_check_fn") + return interrupted + except: + pass + return False + + stop_check_fn = check_comfyui_interrupt + + # Generate with official parameters + with torch.no_grad(): + if use_sampling: + # Use sampling mode (less stable but more varied) + output = self.model.generate( + **inputs, + tokenizer=self.processor.tokenizer, + cfg_scale=cfg_scale, + max_new_tokens=None, + do_sample=True, + temperature=temperature, + top_p=top_p, + stop_check_fn=stop_check_fn, + ) + else: + # Use deterministic mode like official examples + output = self.model.generate( + **inputs, + tokenizer=self.processor.tokenizer, + cfg_scale=cfg_scale, + max_new_tokens=None, + do_sample=False, # More deterministic generation + stop_check_fn=stop_check_fn, + ) + + # Check if we got actual audio output + if hasattr(output, 'speech_outputs') and output.speech_outputs: + speech_tensors = output.speech_outputs + + if isinstance(speech_tensors, list) and len(speech_tensors) > 0: + audio_tensor = torch.cat(speech_tensors, dim=-1) + else: + audio_tensor = speech_tensors + + # Ensure proper format (1, 1, samples) + if audio_tensor.dim() == 1: + audio_tensor = audio_tensor.unsqueeze(0).unsqueeze(0) + elif audio_tensor.dim() == 2: + audio_tensor = audio_tensor.unsqueeze(0) + + # Convert to float32 for compatibility with downstream nodes (Save Audio, etc.) + # Many audio processing nodes don't support BFloat16 + return { + "waveform": audio_tensor.cpu().float(), + "sample_rate": 24000 + } + + elif hasattr(output, 'sequences'): + logger.error("VibeVoice returned only text tokens, no audio generated") + raise Exception("VibeVoice failed to generate audio - only text tokens returned") + + else: + logger.error(f"Unexpected output format from VibeVoice: {type(output)}") + raise Exception(f"VibeVoice returned unexpected output format: {type(output)}") + + except Exception as e: + # Re-raise interruption exceptions without wrapping + import comfy.model_management as mm + if isinstance(e, mm.InterruptProcessingException): + raise # Let the interruption propagate + + # For real errors, log and re-raise with context + logger.error(f"VibeVoice generation failed: {e}") + raise Exception(f"VibeVoice generation failed: {str(e)}") \ No newline at end of file diff --git a/VibeVoice-ComfyUI/nodes/free_memory_node.py b/VibeVoice-ComfyUI/nodes/free_memory_node.py new file mode 100644 index 0000000000000000000000000000000000000000..cca18f82ef4dd960f0861d4215029894f33774ec --- /dev/null +++ b/VibeVoice-ComfyUI/nodes/free_memory_node.py @@ -0,0 +1,110 @@ +# Created by Fabio Sarracino +# Node to free VibeVoice model memory + +import logging +import torch +import gc +from typing import Any + +# Setup logging +logger = logging.getLogger("VibeVoice") + +class VibeVoiceFreeMemoryNode: + """Node to explicitly free VibeVoice model memory""" + + # Class variables to store node instances + _single_speaker_instances = [] + _multi_speaker_instances = [] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio": ("AUDIO", {"tooltip": "Audio input that triggers memory cleanup and gets passed through"}), + } + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "free_vibevoice_memory" + CATEGORY = "VibeVoiceWrapper" + DESCRIPTION = "Free all loaded VibeVoice models from memory when audio passes through" + + @classmethod + def register_single_speaker(cls, node_instance): + """Register a single speaker node instance""" + if node_instance not in cls._single_speaker_instances: + cls._single_speaker_instances.append(node_instance) + + @classmethod + def register_multi_speaker(cls, node_instance): + """Register a multi speaker node instance""" + if node_instance not in cls._multi_speaker_instances: + cls._multi_speaker_instances.append(node_instance) + + def free_vibevoice_memory(self, audio): + """Free memory from all VibeVoice nodes and pass through the audio""" + + try: + freed_count = 0 + + # Try to access and free memory from globally cached instances + # ComfyUI might cache node instances + try: + import sys + from .base_vibevoice import BaseVibeVoiceNode + + # Search in all modules for BaseVibeVoiceNode instances + for module_name, module in sys.modules.items(): + if module and 'vibevoice' in module_name.lower(): + for attr_name in dir(module): + if not attr_name.startswith('_'): + try: + attr = getattr(module, attr_name) + if isinstance(attr, type) and issubclass(attr, BaseVibeVoiceNode): + # Check if the class has any cached instances + for instance_attr in dir(attr): + instance = getattr(attr, instance_attr) + if isinstance(instance, BaseVibeVoiceNode) and hasattr(instance, 'free_memory'): + instance.free_memory() + freed_count += 1 + except: + pass + except: + pass + + # Free from registered single speaker instances + for node in self._single_speaker_instances: + if hasattr(node, 'free_memory'): + node.free_memory() + freed_count += 1 + + # Free from registered multi speaker instances + for node in self._multi_speaker_instances: + if hasattr(node, 'free_memory'): + node.free_memory() + freed_count += 1 + + # Force garbage collection + gc.collect() + + # Clear CUDA cache if available + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.synchronize() + logger.info(f"Freed VibeVoice memory from {freed_count} nodes and cleared CUDA cache") + else: + logger.info(f"Freed VibeVoice memory from {freed_count} nodes") + + # Pass through the audio unchanged + return (audio,) + + except Exception as e: + logger.error(f"Error freeing VibeVoice memory: {str(e)}") + # Still pass through audio even if error occurs + return (audio,) + + @classmethod + def IS_CHANGED(cls, **kwargs): + """Always execute this node""" + return float("nan") # Forces re-execution every time \ No newline at end of file diff --git a/VibeVoice-ComfyUI/nodes/load_text_node.py b/VibeVoice-ComfyUI/nodes/load_text_node.py new file mode 100644 index 0000000000000000000000000000000000000000..5b18f9939eac54689a3a6e5b0262c3f2e154df22 --- /dev/null +++ b/VibeVoice-ComfyUI/nodes/load_text_node.py @@ -0,0 +1,173 @@ +# Created by Fabio Sarracino + +import os +import logging +import hashlib +import folder_paths + +# Setup logging +logger = logging.getLogger("VibeVoice") + +class LoadTextFromFileNode: + @classmethod + def INPUT_TYPES(cls): + # Get all text files from all directories + all_files = [] + + # Add files from each directory with prefix + for dir_name in ["input", "output", "temp"]: + files = cls.get_files_for_directory(dir_name) + for f in files: + if f != "No text files found": + all_files.append(f"{dir_name}/{f}") + + if not all_files: + all_files = ["No text files found in any directory"] + + return { + "required": { + "file": (sorted(all_files), { + "tooltip": "Select a text file to load (format: directory/filename)" + }), + } + } + + @classmethod + def get_files_for_directory(cls, source_dir): + """Get list of text files for the selected directory""" + # Get the appropriate directory path + if source_dir == "input": + dir_path = folder_paths.get_input_directory() + elif source_dir == "output": + dir_path = folder_paths.get_output_directory() + elif source_dir == "temp": + dir_path = folder_paths.get_temp_directory() + else: + return [] + + files = [] + try: + for f in os.listdir(dir_path): + if os.path.isfile(os.path.join(dir_path, f)): + # Check for text file extensions + if f.lower().endswith(('.txt')): + files.append(f) + except Exception as e: + logger.warning(f"Error listing files in {source_dir}: {e}") + + return files + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("text",) + FUNCTION = "load_text" + CATEGORY = "VibeVoiceWrapper" + DESCRIPTION = "Load text content from a .txt file" + + def load_text(self, file: str): + """Load text content from file""" + + try: + # Check if no file selected + if not file or file == "No text files found in any directory": + raise Exception("Please select a valid text file.") + + # Parse directory and filename from the combined string + if "/" not in file: + raise Exception(f"Invalid file format: {file}") + + source_dir, filename = file.split("/", 1) + + # Get the appropriate directory path + if source_dir == "input": + dir_path = folder_paths.get_input_directory() + elif source_dir == "output": + dir_path = folder_paths.get_output_directory() + elif source_dir == "temp": + dir_path = folder_paths.get_temp_directory() + else: + raise Exception(f"Invalid source directory: {source_dir}") + + # Build full file path + file_path = os.path.join(dir_path, filename) + + if not os.path.exists(file_path): + raise Exception(f"File not found: {file_path}") + + # Read file with UTF-8 encoding (most common) + with open(file_path, 'r', encoding='utf-8') as f: + text_content = f.read() + + if not text_content.strip(): + raise Exception("File is empty or contains only whitespace") + + return (text_content,) + + except UnicodeDecodeError as e: + raise Exception(f"Encoding error reading file: {str(e)}. File may not be UTF-8 encoded.") + except Exception as e: + logger.error(f"Failed to load text file: {str(e)}") + raise Exception(f"Error loading text file: {str(e)}") + + @classmethod + def IS_CHANGED(cls, file): + """Cache key for ComfyUI""" + if not file or file == "No text files found in any directory": + return "no_file" + + # Parse directory and filename + if "/" not in file: + return f"{file}_invalid" + + source_dir, filename = file.split("/", 1) + + # Get the appropriate directory path + if source_dir == "input": + dir_path = folder_paths.get_input_directory() + elif source_dir == "output": + dir_path = folder_paths.get_output_directory() + elif source_dir == "temp": + dir_path = folder_paths.get_temp_directory() + else: + return f"{file}_invalid_dir" + + file_path = os.path.join(dir_path, filename) + + if not os.path.exists(file_path): + return f"{file}_not_found" + + # Use file hash for cache invalidation + try: + m = hashlib.sha256() + with open(file_path, 'rb') as f: + m.update(f.read()) + return m.digest().hex() + except: + return f"{file}_error" + + @classmethod + def VALIDATE_INPUTS(cls, file, **kwargs): + """Validate that the file exists""" + if not file or file == "No text files found in any directory": + return "No valid text file selected" + + # Parse directory and filename + if "/" not in file: + return f"Invalid file format: {file}" + + source_dir, filename = file.split("/", 1) + + # Get the appropriate directory path + if source_dir == "input": + dir_path = folder_paths.get_input_directory() + elif source_dir == "output": + dir_path = folder_paths.get_output_directory() + elif source_dir == "temp": + dir_path = folder_paths.get_temp_directory() + else: + return f"Invalid source directory: {source_dir}" + + file_path = os.path.join(dir_path, filename) + if not os.path.exists(file_path): + return f"File not found: {filename} in {source_dir}" + + return True \ No newline at end of file diff --git a/VibeVoice-ComfyUI/nodes/lora_node.py b/VibeVoice-ComfyUI/nodes/lora_node.py new file mode 100644 index 0000000000000000000000000000000000000000..defe446cba4bbcad1e1c9e503c1eb95e6ca9c594 --- /dev/null +++ b/VibeVoice-ComfyUI/nodes/lora_node.py @@ -0,0 +1,214 @@ +# Created by Fabio Sarracino +# Original LoRa code implementation by jpgallegoar-vpai user via PR #127 +# LoRA configuration node for VibeVoice + +import logging +import os +from typing import Dict, Any, List + +# Setup logging +logger = logging.getLogger("VibeVoice") + +# Cache for LoRA scanning to avoid repeated logs +_lora_cache = { + "first_load_logged": False +} + +def get_available_loras() -> List[str]: + """Get list of available LoRA folders in ComfyUI/models/vibevoice/loras""" + try: + import folder_paths + + # Get the ComfyUI models directory + models_dir = folder_paths.get_folder_paths("checkpoints")[0] + # Navigate to vibevoice/loras directory + loras_dir = os.path.join(os.path.dirname(models_dir), "vibevoice", "loras") + + # Create directory if it doesn't exist + os.makedirs(loras_dir, exist_ok=True) + + # List all directories in the loras folder + lora_folders = [] + if os.path.exists(loras_dir): + for item in os.listdir(loras_dir): + item_path = os.path.join(loras_dir, item) + if os.path.isdir(item_path): + # Check if it contains LoRA files + adapter_config = os.path.join(item_path, "adapter_config.json") + adapter_model_st = os.path.join(item_path, "adapter_model.safetensors") + adapter_model_bin = os.path.join(item_path, "adapter_model.bin") + + # Consider it a valid LoRA if it has config or model files + if os.path.exists(adapter_config) or os.path.exists(adapter_model_st) or os.path.exists(adapter_model_bin): + lora_folders.append(item) + + # Only log on first scan to avoid spam + if not _lora_cache["first_load_logged"]: + if not lora_folders: + logger.info("No LoRA adapters found in ComfyUI/models/vibevoice/loras") + _lora_cache["first_load_logged"] = True + + # Always include "None" option to disable LoRA + if not lora_folders: + return ["None"] + + # Sort alphabetically and add None option at the beginning + lora_folders.sort() + return ["None"] + lora_folders + + except Exception as e: + logger.error(f"Error listing LoRA folders: {e}") + return ["None"] + +class VibeVoiceLoRANode: + """Node for configuring LoRA adapters for VibeVoice models""" + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls): + # Get available LoRA folders dynamically + available_loras = get_available_loras() + + return { + "required": { + "lora_name": (available_loras, { + "default": "None", + "tooltip": "Select a LoRA adapter from ComfyUI/models/vibevoice/loras folder" + }), + "llm_strength": ("FLOAT", { + "default": 1.0, + "min": 0.0, + "max": 2.0, + "step": 0.05, + "tooltip": "Strength of the LLM LoRA adapter. Controls how much the LoRA affects the language model" + }), + "use_llm": ("BOOLEAN", { + "default": True, + "tooltip": "Apply LLM (language model) LoRA component when available" + }), + "use_diffusion_head": ("BOOLEAN", { + "default": True, + "tooltip": "Apply diffusion head LoRA/replacement when available" + }), + "use_acoustic_connector": ("BOOLEAN", { + "default": True, + "tooltip": "Apply acoustic connector LoRA component when available" + }), + "use_semantic_connector": ("BOOLEAN", { + "default": True, + "tooltip": "Apply semantic connector LoRA component when available" + }), + } + } + + RETURN_TYPES = ("LORA_CONFIG",) + RETURN_NAMES = ("lora",) + FUNCTION = "configure_lora" + CATEGORY = "VibeVoiceWrapper" + DESCRIPTION = "Configure LoRA adapters for fine-tuned VibeVoice models. Place LoRA folders in ComfyUI/models/vibevoice/loras/" + + def configure_lora(self, lora_name: str = "None", llm_strength: float = 1.0, + use_llm: bool = True, use_diffusion_head: bool = True, + use_acoustic_connector: bool = True, use_semantic_connector: bool = True): + """Configure LoRA settings and validate the path""" + + # Handle "None" selection + if lora_name == "None": + logger.info("No LoRA selected, using base model") + return ({ + "path": None, + "llm_strength": llm_strength, + "use_llm": use_llm, + "use_diffusion_head": use_diffusion_head, + "use_acoustic_connector": use_acoustic_connector, + "use_semantic_connector": use_semantic_connector + },) + + try: + import folder_paths + + # Build full path to the LoRA folder + models_dir = folder_paths.get_folder_paths("checkpoints")[0] + loras_dir = os.path.join(os.path.dirname(models_dir), "vibevoice", "loras") + lora_path = os.path.join(loras_dir, lora_name) + + # Validate the path exists + if not os.path.exists(lora_path): + logger.error(f"LoRA path does not exist: {lora_path}") + raise Exception(f"LoRA folder not found: {lora_name}") + + if not os.path.isdir(lora_path): + logger.error(f"LoRA path is not a directory: {lora_path}") + raise Exception(f"LoRA path must be a directory: {lora_name}") + + # Check for required files + adapter_config = os.path.join(lora_path, "adapter_config.json") + adapter_model_st = os.path.join(lora_path, "adapter_model.safetensors") + adapter_model_bin = os.path.join(lora_path, "adapter_model.bin") + + if not os.path.exists(adapter_config): + logger.warning(f"adapter_config.json not found in {lora_name}") + + if not os.path.exists(adapter_model_st) and not os.path.exists(adapter_model_bin): + logger.warning(f"No adapter model file found in {lora_name}") + logger.warning("Expected: adapter_model.safetensors or adapter_model.bin") + + logger.info(f"LoRA configured: {lora_name} ({lora_path})") + + # Check for optional components + components_found = [] + diffusion_head_path = os.path.join(lora_path, "diffusion_head") + acoustic_connector_path = os.path.join(lora_path, "acoustic_connector") + semantic_connector_path = os.path.join(lora_path, "semantic_connector") + + if os.path.exists(diffusion_head_path): + components_found.append("diffusion_head") + if os.path.exists(acoustic_connector_path): + components_found.append("acoustic_connector") + if os.path.exists(semantic_connector_path): + components_found.append("semantic_connector") + + if components_found: + logger.info(f"Additional LoRA components found: {', '.join(components_found)}") + + # Create configuration dictionary + lora_config = { + "path": lora_path, + "llm_strength": llm_strength, + "use_llm": use_llm, + "use_diffusion_head": use_diffusion_head, + "use_acoustic_connector": use_acoustic_connector, + "use_semantic_connector": use_semantic_connector + } + + # Log configuration + enabled_components = [] + if use_llm: + enabled_components.append(f"LLM (strength: {llm_strength})") + if use_diffusion_head: + enabled_components.append("Diffusion Head") + if use_acoustic_connector: + enabled_components.append("Acoustic Connector") + if use_semantic_connector: + enabled_components.append("Semantic Connector") + + if enabled_components: + logger.info(f"LoRA components enabled: {', '.join(enabled_components)}") + else: + logger.warning("All LoRA components are disabled") + + return (lora_config,) + + except ImportError: + logger.error("Could not import folder_paths from ComfyUI") + raise Exception("Failed to access ComfyUI folders") + except Exception as e: + logger.error(f"Error configuring LoRA: {e}") + raise + + @classmethod + def IS_CHANGED(cls, lora_name: str = "None", **kwargs): + """Cache key for ComfyUI - includes all parameters""" + return f"{lora_name}_{kwargs.get('llm_strength', 1.0)}_{kwargs.get('use_llm', True)}_{kwargs.get('use_diffusion_head', True)}_{kwargs.get('use_acoustic_connector', True)}_{kwargs.get('use_semantic_connector', True)}" \ No newline at end of file diff --git a/VibeVoice-ComfyUI/nodes/multi_speaker_node.py b/VibeVoice-ComfyUI/nodes/multi_speaker_node.py new file mode 100644 index 0000000000000000000000000000000000000000..f0eb3825a341f06c961ac7aabb415ba20b65cf8b --- /dev/null +++ b/VibeVoice-ComfyUI/nodes/multi_speaker_node.py @@ -0,0 +1,392 @@ +# Created by Fabio Sarracino + +import logging +import os +import re +import tempfile +import torch +import numpy as np +from typing import List, Optional + +from .base_vibevoice import BaseVibeVoiceNode, get_available_models + +# Setup logging +logger = logging.getLogger("VibeVoice") + +class VibeVoiceMultipleSpeakersNode(BaseVibeVoiceNode): + def __init__(self): + super().__init__() + # Register this instance for memory management + try: + from .free_memory_node import VibeVoiceFreeMemoryNode + VibeVoiceFreeMemoryNode.register_multi_speaker(self) + except: + pass + + @classmethod + def INPUT_TYPES(cls): + # Get available models dynamically + available_models = get_available_models() + model_choices = [display_name for _, display_name in available_models] + # Try to select Large model by default if available + default_model = "VibeVoice-Large" + if default_model not in model_choices: + default_model = model_choices[0] if model_choices else "No models found" + + return { + "required": { + "text": ("STRING", { + "multiline": True, + "default": "[1]: Hello, this is the first speaker.\n[2]: Hi there, I'm the second speaker.\n[1]: Nice to meet you!\n[2]: Nice to meet you too!", + "tooltip": "Text with speaker labels. Use '[N]:' format where N is 1-4. Gets disabled when connected to another node.", + "forceInput": False, + "dynamicPrompts": True + }), + "model": (model_choices if model_choices else ["No models found"], { + "default": default_model, + "tooltip": "Select a model from ComfyUI/models/vibevoice/ folder. Large is recommended for multi-speaker" + }), + "attention_type": (["auto", "eager", "sdpa", "flash_attention_2", "sage"], { + "default": "auto", + "tooltip": "Attention implementation. Auto selects the best available, eager is standard, sdpa is optimized PyTorch, flash_attention_2 requires compatible GPU, sage uses quantized attention for speedup (CUDA only)" + }), + "quantize_llm": (["full precision", "4bit", "8bit"], { + "default": "full precision", + "tooltip": "Dynamically quantize only the LLM component for non-quantized models. 4bit: major VRAM savings with minimal quality loss. 8bit: good balance of quality and memory usage. Full precision: original quality. Note: ignored for pre-quantized models. Requires CUDA GPU." + }), + "free_memory_after_generate": ("BOOLEAN", {"default": True, "tooltip": "Free model from memory after generation to save VRAM/RAM. Disable to keep model loaded for faster subsequent generations"}), + "diffusion_steps": ("INT", {"default": 20, "min": 1, "max": 100, "step": 1, "tooltip": "Number of denoising steps. More steps = theoretically better quality but slower. Default: 20"}), + "seed": ("INT", {"default": 42, "min": 0, "max": 2**32-1, "tooltip": "Random seed for generation. Default 42 is used in official examples"}), + "cfg_scale": ("FLOAT", {"default": 1.3, "min": 0.5, "max": 3.5, "step": 0.05, "tooltip": "Classifier-free guidance scale (official default: 1.3)"}), + "use_sampling": ("BOOLEAN", {"default": False, "tooltip": "Enable sampling mode. When False (default), uses deterministic generation like official examples"}), + }, + "optional": { + "speaker1_voice": ("AUDIO", {"tooltip": "Optional: Voice sample for Speaker 1. If not provided, synthetic voice will be used."}), + "speaker2_voice": ("AUDIO", {"tooltip": "Optional: Voice sample for Speaker 2. If not provided, synthetic voice will be used."}), + "speaker3_voice": ("AUDIO", {"tooltip": "Optional: Voice sample for Speaker 3. If not provided, synthetic voice will be used."}), + "speaker4_voice": ("AUDIO", {"tooltip": "Optional: Voice sample for Speaker 4. If not provided, synthetic voice will be used."}), + "lora": ("LORA_CONFIG", {"tooltip": "Optional: LoRA configuration from VibeVoice LoRA node"}), + "temperature": ("FLOAT", {"default": 0.95, "min": 0.1, "max": 2.0, "step": 0.05, "tooltip": "Only used when sampling is enabled"}), + "top_p": ("FLOAT", {"default": 0.95, "min": 0.1, "max": 1.0, "step": 0.05, "tooltip": "Only used when sampling is enabled"}), + "voice_speed_factor": ("FLOAT", { + "default": 1.0, + "min": 0.8, + "max": 1.2, + "step": 0.01, + "tooltip": "1.0 = normal speed, <1.0 = slower speed, >1.0 = faster speed (applies to all speakers)" + }), + } + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "generate_speech" + CATEGORY = "VibeVoiceWrapper" + DESCRIPTION = "Generate multi-speaker conversations with up to 4 distinct voices using Microsoft VibeVoice" + + def _prepare_voice_sample(self, voice_audio, speaker_idx: int, voice_speed_factor: float = 1.0) -> Optional[np.ndarray]: + """Prepare a single voice sample from input audio with speed adjustment""" + return self._prepare_audio_from_comfyui(voice_audio, speed_factor=voice_speed_factor) + + def generate_speech(self, text: str = "", model: str = "VibeVoice-7B-Preview", + attention_type: str = "auto", quantize_llm: str = "full precision", + free_memory_after_generate: bool = True, + diffusion_steps: int = 20, seed: int = 42, cfg_scale: float = 1.3, + use_sampling: bool = False, lora=None, + speaker1_voice=None, speaker2_voice=None, + speaker3_voice=None, speaker4_voice=None, + temperature: float = 0.95, top_p: float = 0.95, + voice_speed_factor: float = 1.0): + """Generate multi-speaker speech from text using VibeVoice""" + + try: + # Check text input + if not text or not text.strip(): + raise Exception("No text provided. Please enter text with speaker labels (e.g., '[1]: Hello' or '[2]: Hi')") + + # First detect how many speakers are in the text + bracket_pattern = r'\[(\d+)\]\s*:' + speakers_numbers = sorted(list(set([int(m) for m in re.findall(bracket_pattern, text)]))) + + # Limit to 1-4 speakers + if not speakers_numbers: + num_speakers = 1 # Default to 1 if no speaker format found + else: + num_speakers = min(max(speakers_numbers), 4) # Max speaker number, capped at 4 + if max(speakers_numbers) > 4: + print(f"[VibeVoice] Warning: Found {max(speakers_numbers)} speakers, limiting to 4") + + # Direct conversion from [N]: to Speaker (N-1): for VibeVoice processor + # This avoids multiple conversion steps + converted_text = text + + # Find all [N]: patterns in the text + speakers_in_text = sorted(list(set([int(m) for m in re.findall(bracket_pattern, text)]))) + + if not speakers_in_text: + # No [N]: format found, try Speaker N: format + speaker_pattern = r'Speaker\s+(\d+)\s*:' + speakers_in_text = sorted(list(set([int(m) for m in re.findall(speaker_pattern, text)]))) + + if speakers_in_text: + # Text already in Speaker N format, convert to 0-based + for speaker_num in sorted(speakers_in_text, reverse=True): + pattern = f'Speaker\\s+{speaker_num}\\s*:' + replacement = f'Speaker {speaker_num - 1}:' + converted_text = re.sub(pattern, replacement, converted_text) + else: + # No speaker format found + speakers_in_text = [1] + + # Parse pause keywords even for single speaker + pause_segments = self._parse_pause_keywords(text) + + # Store speaker segments for pause processing + speaker_segments_with_pauses = [] + segments = [] + + for seg_type, seg_content in pause_segments: + if seg_type == 'pause': + speaker_segments_with_pauses.append(('pause', seg_content, None)) + else: + # Clean up newlines + text_clean = seg_content.replace('\n', ' ').replace('\r', ' ') + text_clean = ' '.join(text_clean.split()) + + if text_clean: + speaker_segments_with_pauses.append(('text', text_clean, 1)) + segments.append(f"Speaker 0: {text_clean}") + + # Join all segments for fallback + converted_text = '\n'.join(segments) if segments else f"Speaker 0: {text}" + else: + # Convert [N]: directly to Speaker (N-1): and handle multi-line text + # Split text to preserve speaker segments while cleaning up newlines within each segment + segments = [] + + # Find all speaker markers with their positions + speaker_matches = list(re.finditer(f'\\[({"|".join(map(str, speakers_in_text))})\\]\\s*:', converted_text)) + + # Store speaker segments for pause processing + speaker_segments_with_pauses = [] + + for i, match in enumerate(speaker_matches): + speaker_num = int(match.group(1)) + start = match.end() + + # Find where this speaker's text ends (at next speaker or end of text) + if i + 1 < len(speaker_matches): + end = speaker_matches[i + 1].start() + else: + end = len(converted_text) + + # Extract the speaker's text (keep pause keywords for now) + speaker_text = converted_text[start:end].strip() + + # Parse pause keywords within this speaker's text + pause_segments = self._parse_pause_keywords(speaker_text) + + # Process each segment (text or pause) for this speaker + for seg_type, seg_content in pause_segments: + if seg_type == 'pause': + # Add pause segment + speaker_segments_with_pauses.append(('pause', seg_content, None)) + else: + # Clean up the text segment + text_clean = seg_content.replace('\n', ' ').replace('\r', ' ') + text_clean = ' '.join(text_clean.split()) + + if text_clean: # Only add non-empty text + # Add text segment with speaker info + speaker_segments_with_pauses.append(('text', text_clean, speaker_num)) + # Also build the traditional segments for fallback + segments.append(f'Speaker {speaker_num - 1}: {text_clean}') + + # Join all segments with newlines (required for multi-speaker format) - for fallback + converted_text = '\n'.join(segments) if segments else "" + + # Build speaker names list - these are just for logging, not used by processor + # The processor uses the speaker labels in the text itself + speakers = [f"Speaker {i}" for i in range(len(speakers_in_text))] + + # Get the actual folder path for the selected model + available_models = get_available_models() + model_path = None + for folder, display_name in available_models: + if display_name == model: + model_path = folder + break + + if not model_path: + raise Exception(f"Model '{model}' not found in models/vibevoice/") + + # Extract LoRA configuration if provided + lora_path = None + llm_lora_strength = 1.0 + if lora and isinstance(lora, dict): + lora_path = lora.get("path", None) + llm_lora_strength = lora.get("llm_strength", 1.0) + + # Set LoRA component flags based on configuration + self.use_llm_lora = lora.get("use_llm", True) + self.use_diffusion_head_lora = lora.get("use_diffusion_head", True) + self.use_acoustic_connector_lora = lora.get("use_acoustic_connector", True) + self.use_semantic_connector_lora = lora.get("use_semantic_connector", True) + + if lora_path: + logger.info(f"Using LoRA from: {lora_path}") + + # Load model with optional LoRA + self.load_model(model, model_path, attention_type, quantize_llm=quantize_llm, lora_path=lora_path) + + voice_inputs = [speaker1_voice, speaker2_voice, speaker3_voice, speaker4_voice] + + # Prepare voice samples in order of appearance + voice_samples = [] + for i, speaker_num in enumerate(speakers_in_text): + idx = speaker_num - 1 # Convert to 0-based for voice array + + # Try to use provided voice sample + if idx < len(voice_inputs) and voice_inputs[idx] is not None: + voice_sample = self._prepare_voice_sample(voice_inputs[idx], idx, voice_speed_factor) + if voice_sample is None: + # Use the actual speaker index for consistent synthetic voice + voice_sample = self._create_synthetic_voice_sample(idx) + else: + # Use the actual speaker index for consistent synthetic voice + voice_sample = self._create_synthetic_voice_sample(idx) + + voice_samples.append(voice_sample) + + # Ensure voice_samples count matches detected speakers + if len(voice_samples) != len(speakers_in_text): + logger.error(f"Mismatch: {len(speakers_in_text)} speakers but {len(voice_samples)} voice samples!") + raise Exception(f"Voice sample count mismatch: expected {len(speakers_in_text)}, got {len(voice_samples)}") + + # Check if we have pause segments to process + if 'speaker_segments_with_pauses' in locals() and speaker_segments_with_pauses: + # Process segments with pauses + all_audio_segments = [] + sample_rate = 24000 # VibeVoice uses 24kHz + + # Group consecutive text segments from same speaker for efficiency + grouped_segments = [] + current_group = [] + current_speaker = None + + for seg_type, seg_content, speaker_num in speaker_segments_with_pauses: + if seg_type == 'pause': + # Save current group if any + if current_group: + grouped_segments.append(('text_group', current_group, current_speaker)) + current_group = [] + current_speaker = None + # Add pause + grouped_segments.append(('pause', seg_content, None)) + else: + # Text segment + if speaker_num == current_speaker: + # Same speaker, add to current group + current_group.append(seg_content) + else: + # Different speaker, save current group and start new one + if current_group: + grouped_segments.append(('text_group', current_group, current_speaker)) + current_group = [seg_content] + current_speaker = speaker_num + + # Save last group if any + if current_group: + grouped_segments.append(('text_group', current_group, current_speaker)) + + # Process grouped segments + for seg_type, seg_content, speaker_num in grouped_segments: + if seg_type == 'pause': + # Generate silence + duration_ms = seg_content + logger.info(f"Adding {duration_ms}ms pause") + silence_audio = self._generate_silence(duration_ms, sample_rate) + all_audio_segments.append(silence_audio) + else: + # Process text group for a speaker + combined_text = ' '.join(seg_content) + formatted_text = f"Speaker {speaker_num - 1}: {combined_text}" + + # Get voice sample for this speaker + speaker_idx = speakers_in_text.index(speaker_num) + speaker_voice_samples = [voice_samples[speaker_idx]] + + logger.info(f"Generating audio for Speaker {speaker_num}: {len(combined_text.split())} words") + + # Generate audio for this speaker's text + segment_audio = self._generate_with_vibevoice( + formatted_text, speaker_voice_samples, cfg_scale, seed, + diffusion_steps, use_sampling, temperature, top_p, + llm_lora_strength=llm_lora_strength + ) + + all_audio_segments.append(segment_audio) + + # Concatenate all audio segments + if all_audio_segments: + logger.info(f"Concatenating {len(all_audio_segments)} audio segments (including pauses)...") + + # Extract waveforms + waveforms = [] + for audio_segment in all_audio_segments: + if isinstance(audio_segment, dict) and "waveform" in audio_segment: + waveforms.append(audio_segment["waveform"]) + + if waveforms: + # Filter out None values if any + valid_waveforms = [w for w in waveforms if w is not None] + + if valid_waveforms: + # Concatenate along time dimension + combined_waveform = torch.cat(valid_waveforms, dim=-1) + + audio_dict = { + "waveform": combined_waveform, + "sample_rate": sample_rate + } + logger.info(f"Successfully generated multi-speaker audio with pauses") + else: + raise Exception("No valid audio waveforms generated") + else: + raise Exception("Failed to extract waveforms from audio segments") + else: + raise Exception("No audio segments generated") + else: + # Fallback to original method without pause support + logger.info("Processing without pause support (no pause keywords found)") + audio_dict = self._generate_with_vibevoice( + converted_text, voice_samples, cfg_scale, seed, diffusion_steps, + use_sampling, temperature, top_p, llm_lora_strength=llm_lora_strength + ) + + # Free memory if requested + if free_memory_after_generate: + self.free_memory() + + return (audio_dict,) + + except Exception as e: + # Check if this is an interruption by the user + import comfy.model_management as mm + if isinstance(e, mm.InterruptProcessingException): + # User interrupted - just log it and re-raise to stop the workflow + logger.info("Generation interrupted by user") + raise # Propagate the interruption to stop the workflow + else: + # Real error - show it + logger.error(f"Multi-speaker speech generation failed: {str(e)}") + raise Exception(f"Error generating multi-speaker speech: {str(e)}") + + @classmethod + def IS_CHANGED(cls, text="", model="VibeVoice-7B-Preview", + speaker1_voice=None, speaker2_voice=None, + speaker3_voice=None, speaker4_voice=None, lora=None, **kwargs): + """Cache key for ComfyUI""" + voices_hash = hash(str([speaker1_voice, speaker2_voice, speaker3_voice, speaker4_voice])) + lora_hash = hash(str(lora)) if lora else 0 + return f"{hash(text)}_{model}_{voices_hash}_{lora_hash}_{kwargs.get('cfg_scale', 1.3)}_{kwargs.get('seed', 0)}" \ No newline at end of file diff --git a/VibeVoice-ComfyUI/nodes/single_speaker_node.py b/VibeVoice-ComfyUI/nodes/single_speaker_node.py new file mode 100644 index 0000000000000000000000000000000000000000..d1c7aa74d6776fc50b3fa204afa11af7a798807a --- /dev/null +++ b/VibeVoice-ComfyUI/nodes/single_speaker_node.py @@ -0,0 +1,265 @@ +# Created by Fabio Sarracino + +import logging +import os +import tempfile +import torch +import numpy as np +import re +from typing import List, Optional + +from .base_vibevoice import BaseVibeVoiceNode, get_available_models + +# Setup logging +logger = logging.getLogger("VibeVoice") + +class VibeVoiceSingleSpeakerNode(BaseVibeVoiceNode): + def __init__(self): + super().__init__() + # Register this instance for memory management + try: + from .free_memory_node import VibeVoiceFreeMemoryNode + VibeVoiceFreeMemoryNode.register_single_speaker(self) + except: + pass + + @classmethod + def INPUT_TYPES(cls): + # Get available models dynamically + available_models = get_available_models() + model_choices = [display_name for _, display_name in available_models] + default_model = model_choices[0] if model_choices else "No models found" + + return { + "required": { + "text": ("STRING", { + "multiline": True, + "default": "Hello, this is a test of the VibeVoice text-to-speech system.", + "tooltip": "Text to convert to speech. Gets disabled when connected to another node.", + "forceInput": False, + "dynamicPrompts": True + }), + "model": (model_choices if model_choices else ["No models found"], { + "default": default_model, + "tooltip": "Select a model from ComfyUI/models/vibevoice/ folder" + }), + "attention_type": (["auto", "eager", "sdpa", "flash_attention_2", "sage"], { + "default": "auto", + "tooltip": "Attention implementation. Auto selects the best available, eager is standard, sdpa is optimized PyTorch, flash_attention_2 requires compatible GPU, sage uses quantized attention for speedup (CUDA only)" + }), + "quantize_llm": (["full precision", "4bit", "8bit"], { + "default": "full precision", + "tooltip": "Dynamically quantize only the LLM component for non-quantized models. 4bit: major VRAM savings with minimal quality loss. 8bit: good balance of quality and memory usage. Full precision: original quality. Note: ignored for pre-quantized models. Requires CUDA GPU." + }), + "free_memory_after_generate": ("BOOLEAN", {"default": True, "tooltip": "Free model from memory after generation to save VRAM/RAM. Disable to keep model loaded for faster subsequent generations"}), + "diffusion_steps": ("INT", {"default": 20, "min": 1, "max": 100, "step": 1, "tooltip": "Number of denoising steps. More steps = theoretically better quality but slower. Default: 20"}), + "seed": ("INT", {"default": 42, "min": 0, "max": 2**32-1, "tooltip": "Random seed for generation. Default 42 is used in official examples"}), + "cfg_scale": ("FLOAT", {"default": 1.3, "min": 0.5, "max": 3.5, "step": 0.05, "tooltip": "Classifier-free guidance scale (official default: 1.3)"}), + "use_sampling": ("BOOLEAN", {"default": False, "tooltip": "Enable sampling mode. When False (default), uses deterministic generation like official examples"}), + }, + "optional": { + "voice_to_clone": ("AUDIO", {"tooltip": "Optional: Reference voice to clone. If not provided, synthetic voice will be used."}), + "lora": ("LORA_CONFIG", {"tooltip": "Optional: LoRA configuration from VibeVoice LoRA node"}), + "temperature": ("FLOAT", {"default": 0.95, "min": 0.1, "max": 2.0, "step": 0.05, "tooltip": "Only used when sampling is enabled"}), + "top_p": ("FLOAT", {"default": 0.95, "min": 0.1, "max": 1.0, "step": 0.05, "tooltip": "Only used when sampling is enabled"}), + "max_words_per_chunk": ("INT", {"default": 250, "min": 100, "max": 500, "step": 50, "tooltip": "Maximum words per chunk for long texts. Lower values prevent speed issues but create more chunks."}), + "voice_speed_factor": ("FLOAT", { + "default": 1.0, + "min": 0.8, + "max": 1.2, + "step": 0.01, + "tooltip": "1.0 = normal speed, <1.0 = slower speed, >1.0 = faster speed" + }), + } + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "generate_speech" + CATEGORY = "VibeVoiceWrapper" + DESCRIPTION = "Generate speech from text using Microsoft VibeVoice with optional voice cloning" + + def _prepare_voice_samples(self, speakers: list, voice_to_clone, voice_speed_factor: float = 1.0) -> List[np.ndarray]: + """Prepare voice samples from input audio or create synthetic ones""" + + if voice_to_clone is not None: + # Use the base class method to prepare audio with speed adjustment + audio_np = self._prepare_audio_from_comfyui(voice_to_clone, speed_factor=voice_speed_factor) + if audio_np is not None: + return [audio_np] + + # Create synthetic voice samples for speakers + voice_samples = [] + for i, speaker in enumerate(speakers): + voice_sample = self._create_synthetic_voice_sample(i) + voice_samples.append(voice_sample) + + return voice_samples + + def generate_speech(self, text: str = "", model: str = "VibeVoice-1.5B", + attention_type: str = "auto", quantize_llm: str = "full precision", + free_memory_after_generate: bool = True, + diffusion_steps: int = 20, seed: int = 42, cfg_scale: float = 1.3, + use_sampling: bool = False, voice_to_clone=None, lora=None, + temperature: float = 0.95, top_p: float = 0.95, + max_words_per_chunk: int = 250, voice_speed_factor: float = 1.0): + """Generate speech from text using VibeVoice""" + + try: + # Use text directly (it now serves as both manual input and connection input) + if text and text.strip(): + final_text = text + else: + raise Exception("No text provided. Please enter text or connect from LoadTextFromFile node.") + + # Get the actual folder path for the selected model + available_models = get_available_models() + model_path = None + for folder, display_name in available_models: + if display_name == model: + model_path = folder + break + + if not model_path: + raise Exception(f"Model '{model}' not found in models/vibevoice/") + + # Extract LoRA configuration if provided + lora_path = None + llm_lora_strength = 1.0 + if lora and isinstance(lora, dict): + lora_path = lora.get("path", None) + llm_lora_strength = lora.get("llm_strength", 1.0) + + # Set LoRA component flags based on configuration + self.use_llm_lora = lora.get("use_llm", True) + self.use_diffusion_head_lora = lora.get("use_diffusion_head", True) + self.use_acoustic_connector_lora = lora.get("use_acoustic_connector", True) + self.use_semantic_connector_lora = lora.get("use_semantic_connector", True) + + if lora_path: + logger.info(f"Using LoRA from: {lora_path}") + + # Load model with optional LoRA + self.load_model(model, model_path, attention_type, quantize_llm=quantize_llm, lora_path=lora_path) + + # For single speaker, we just use ["Speaker 1"] + speakers = ["Speaker 1"] + + # Parse pause keywords from text + segments = self._parse_pause_keywords(final_text) + + # Process segments + all_audio_segments = [] + voice_samples = None # Will be created on first text segment + sample_rate = 24000 # VibeVoice uses 24kHz + + for seg_idx, (seg_type, seg_content) in enumerate(segments): + if seg_type == 'pause': + # Generate silence for pause + duration_ms = seg_content + logger.info(f"Adding {duration_ms}ms pause") + silence_audio = self._generate_silence(duration_ms, sample_rate) + all_audio_segments.append(silence_audio) + + elif seg_type == 'text': + # Process text segment (with chunking if needed) + word_count = len(seg_content.split()) + + if word_count > max_words_per_chunk: + # Split long text into chunks + logger.info(f"Text segment {seg_idx+1} has {word_count} words, splitting into chunks...") + text_chunks = self._split_text_into_chunks(seg_content, max_words_per_chunk) + + for chunk_idx, chunk in enumerate(text_chunks): + logger.info(f"Processing chunk {chunk_idx+1}/{len(text_chunks)} of segment {seg_idx+1}...") + + # Format chunk for VibeVoice + formatted_text = self._format_text_for_vibevoice(chunk, speakers) + + # Create voice samples on first text segment + if voice_samples is None: + voice_samples = self._prepare_voice_samples(speakers, voice_to_clone, voice_speed_factor) + + # Generate audio for this chunk + chunk_audio = self._generate_with_vibevoice( + formatted_text, voice_samples, cfg_scale, + seed, # Use same seed for voice consistency + diffusion_steps, use_sampling, temperature, top_p, + llm_lora_strength=llm_lora_strength + ) + + all_audio_segments.append(chunk_audio) + else: + # Process as single chunk + logger.info(f"Processing text segment {seg_idx+1} ({word_count} words)") + + # Format text for VibeVoice + formatted_text = self._format_text_for_vibevoice(seg_content, speakers) + + # Create voice samples on first text segment + if voice_samples is None: + voice_samples = self._prepare_voice_samples(speakers, voice_to_clone, voice_speed_factor) + + # Generate audio + segment_audio = self._generate_with_vibevoice( + formatted_text, voice_samples, cfg_scale, seed, diffusion_steps, + use_sampling, temperature, top_p, llm_lora_strength=llm_lora_strength + ) + + all_audio_segments.append(segment_audio) + + # Concatenate all audio segments (including pauses) + if all_audio_segments: + logger.info(f"Concatenating {len(all_audio_segments)} audio segments (including pauses)...") + + # Extract waveforms from all segments + waveforms = [] + for audio_segment in all_audio_segments: + if isinstance(audio_segment, dict) and "waveform" in audio_segment: + waveforms.append(audio_segment["waveform"]) + + if waveforms: + # Filter out None values if any + valid_waveforms = [w for w in waveforms if w is not None] + + if valid_waveforms: + # Concatenate along the time dimension (last dimension) + combined_waveform = torch.cat(valid_waveforms, dim=-1) + + # Create final audio dict + audio_dict = { + "waveform": combined_waveform, + "sample_rate": sample_rate + } + logger.info(f"Successfully generated audio with {len(segments)} segments") + else: + raise Exception("No valid audio waveforms generated") + else: + raise Exception("Failed to extract waveforms from audio segments") + else: + raise Exception("No audio segments generated") + + # Free memory if requested + if free_memory_after_generate: + self.free_memory() + + return (audio_dict,) + + except Exception as e: + # Check if this is an interruption by the user + import comfy.model_management as mm + if isinstance(e, mm.InterruptProcessingException): + # User interrupted - just log it and re-raise to stop the workflow + logger.info("Generation interrupted by user") + raise # Propagate the interruption to stop the workflow + else: + # Real error - show it + logger.error(f"Single speaker speech generation failed: {str(e)}") + raise Exception(f"Error generating speech: {str(e)}") + + @classmethod + def IS_CHANGED(cls, text="", model="VibeVoice-1.5B", voice_to_clone=None, lora=None, **kwargs): + """Cache key for ComfyUI""" + voice_hash = hash(str(voice_to_clone)) if voice_to_clone else 0 + lora_hash = hash(str(lora)) if lora else 0 + return f"{hash(text)}_{model}_{voice_hash}_{lora_hash}_{kwargs.get('cfg_scale', 1.3)}_{kwargs.get('seed', 0)}" \ No newline at end of file diff --git a/VibeVoice-ComfyUI/pyproject.toml b/VibeVoice-ComfyUI/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..57ce71756f59110df744b34547f31891375c0b9b --- /dev/null +++ b/VibeVoice-ComfyUI/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "VibeVoice-ComfyUI" +version = "1.8.1" +description = "ComfyUI wrapper for Microsoft VibeVoice TTS model. Supports single speaker, multi-speaker, and text file loading" +license = {file = "LICENSE"} +authors = [{name = "Fabio Sarracino"}] +dependencies = ["accelerate>=1.6.0", "transformers>=4.51.3", "diffusers", "tqdm", "scipy", "ml-collections", "torch>=2.0.0", "torchaudio>=2.0.0", "numpy>=1.20.0", "librosa>=0.9.0", "soundfile>=0.12.0", "av>=14.3.0", "peft>=0.17.0", "huggingface_hub>=0.25.1", "absl-py", "aiortc", "bitsandbytes>=0.48.1", "protobuf"] + +[project.urls] +Repository = "https://github.com/Enemyx-net/VibeVoice-ComfyUI" +"Bug Tracker" = "https://github.com/Enemyx-net/VibeVoice-ComfyUI/issues" + +[tool.comfy] +PublisherId = "enemyx" +DisplayName = "VibeVoice ComfyUI" +Icon = "" +includes = [] + diff --git a/VibeVoice-ComfyUI/requirements.txt b/VibeVoice-ComfyUI/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b73f5fd1080972a1db03b5bd35b678e56abe106 --- /dev/null +++ b/VibeVoice-ComfyUI/requirements.txt @@ -0,0 +1,18 @@ +accelerate>=1.6.0 +transformers>=4.51.3,<5.0.0 +diffusers +tqdm +scipy +ml-collections +torch>=2.0.0 +torchaudio>=2.0.0 +numpy>=1.20.0 +librosa>=0.9.0 +soundfile>=0.12.0 +av>=14.3.0 +peft>=0.17.0 +huggingface_hub>=0.25.1 +absl-py +aiortc +bitsandbytes>=0.48.1 +protobuf diff --git a/VibeVoice-ComfyUI/vvembed/LICENSE b/VibeVoice-ComfyUI/vvembed/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9ff4fa548bd58f5a685f4b8b0e1d6d6884b92b6e --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/LICENSE @@ -0,0 +1,26 @@ +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- + +This is the original VibeVoice code from Microsoft, embedded here as the +repository has been removed from GitHub. The code is used under the MIT license. \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/README.md b/VibeVoice-ComfyUI/vvembed/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0a9709282b1016dfe5139336f98f2be1ddcaadac --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/README.md @@ -0,0 +1,21 @@ +# Embedded VibeVoice + +This folder contains the embedded VibeVoice code from Microsoft. + +## Why Embedded? + +The original VibeVoice repository (https://github.com/microsoft/VibeVoice) has been removed from GitHub. Since VibeVoice is licensed under MIT, we have embedded the code here to ensure continued functionality of the ComfyUI wrapper. + +## License + +The code in this folder is licensed under the MIT License (see LICENSE file). Original copyright belongs to Microsoft Corporation. + +## Modifications + +The only modifications made to the original code are: +- Changed absolute imports from `vibevoice` to relative imports +- No functional changes to the core logic + +## Note + +This is a preservation copy to ensure the continued availability of VibeVoice for the ComfyUI community. diff --git a/VibeVoice-ComfyUI/vvembed/__init__.py b/VibeVoice-ComfyUI/vvembed/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VibeVoice-ComfyUI/vvembed/modular/__init__.py b/VibeVoice-ComfyUI/vvembed/modular/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/__init__.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e81d9c0098bdb5539030f301264479a2e6aadca Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/__init__.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/configuration_vibevoice.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/configuration_vibevoice.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6162850cc3df2b5d462269f08a8eb706023d01b2 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/configuration_vibevoice.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modeling_vibevoice.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modeling_vibevoice.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4d6062cb8070dd191da23b36a76650d9fbfceb5 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modeling_vibevoice.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modeling_vibevoice_inference.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modeling_vibevoice_inference.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92c506ba2d90d7a9d59683d924cff5777cd8e170 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modeling_vibevoice_inference.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_diffusion_head.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_diffusion_head.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b6bc609588e805ea97ca77919a0c0394690c603 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_diffusion_head.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_text_tokenizer.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_text_tokenizer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..216dbfa5a9148e020731029261efa0486fa917c2 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_text_tokenizer.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_tokenizer.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_tokenizer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4bc723859889a684728f6d95b4f34ae31a80ce13 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/modular_vibevoice_tokenizer.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/__pycache__/streamer.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/streamer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..256cc92706ec9d2428281376cefc5218304f90a1 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/modular/__pycache__/streamer.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/modular/configuration_vibevoice.py b/VibeVoice-ComfyUI/vvembed/modular/configuration_vibevoice.py new file mode 100644 index 0000000000000000000000000000000000000000..e577fc2eab4d4bd3aaeab6234bb7bf485054c6f0 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/modular/configuration_vibevoice.py @@ -0,0 +1,270 @@ +# Original code by Microsoft +# updated by Fabio Sarracino - Enemyx-net + +""" VibeVoice_AcousticTokenizer model configuration""" + +from typing import Dict, List, Optional, Tuple + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +from transformers.models.qwen2.configuration_qwen2 import Qwen2Config + +logger = logging.get_logger(__name__) + +# to be improved... + + +class VibeVoiceAcousticTokenizerConfig(PretrainedConfig): + model_type = "vibevoice_acoustic_tokenizer" + + def __init__( + self, + channels: int = 1, + corpus_normalize: float = 0.0, + causal: bool = True, + vae_dim: int = 64, + fix_std: float = 0.5, + std_dist_type: str = 'gaussian', + # common + mixer_layer: str = 'depthwise_conv', + conv_norm: str = 'none', + pad_mode: str = 'constant', + disable_last_norm: bool = True, + layernorm: str = 'RMSNorm', + layernorm_eps: float = 1e-5, + layernorm_elementwise_affine: bool = True, + conv_bias: bool = True, + layer_scale_init_value: float = 1e-6, + weight_init_value: float = 1e-2, + # encoder specific + encoder_n_filters: int = 32, + encoder_ratios: Optional[List[int]] = [8,5,5,4,2,2], + encoder_depths: str = "3-3-3-3-3-3-8", + # decoder specific + decoder_n_filters: int = 32, + decoder_ratios: Optional[List[int]] = None, # if None, same as encoder + decoder_depths: Optional[str] = None, + **kwargs + ): + super().__init__(**kwargs) + self.channels = channels + self.corpus_normalize = corpus_normalize + self.causal = causal + self.vae_dim = vae_dim + self.fix_std = fix_std + self.std_dist_type = std_dist_type + + # common parameters + self.conv_norm = conv_norm + self.pad_mode = pad_mode + self.layernorm_eps = layernorm_eps + self.disable_last_norm = disable_last_norm + self.layernorm = layernorm + self.layernorm_elementwise_affine = layernorm_elementwise_affine + self.conv_bias = conv_bias + self.layer_scale_init_value = layer_scale_init_value + self.weight_init_value = weight_init_value + self.mixer_layer = mixer_layer + + # encoder specific parameters + self.encoder_n_filters = encoder_n_filters + self.encoder_ratios = encoder_ratios + self.encoder_depths = encoder_depths + + # decoder specific parameters + self.decoder_ratios = decoder_ratios if decoder_ratios is not None else encoder_ratios + self.decoder_n_filters = decoder_n_filters + self.decoder_depths = decoder_depths + + +class VibeVoiceSemanticTokenizerConfig(PretrainedConfig): + model_type = "vibevoice_semantic_tokenizer" + + def __init__( + self, + channels: int = 1, + corpus_normalize: float = 0.0, + causal: bool = True, + vae_dim: int = 64, + fix_std: float = 0, + std_dist_type: str = 'none', + # common + mixer_layer: str = 'depthwise_conv', + conv_norm: str = 'none', + pad_mode: str = 'constant', + disable_last_norm: bool = True, + layernorm: str = 'RMSNorm', + layernorm_eps: float = 1e-5, + layernorm_elementwise_affine: bool = True, + conv_bias: bool = True, + layer_scale_init_value: float = 1e-6, + weight_init_value: float = 1e-2, + # encoder specific + encoder_n_filters: int = 32, + encoder_ratios: Optional[List[int]] = [8,5,5,4,2,2], + encoder_depths: str = "3-3-3-3-3-3-8", + **kwargs + ): + super().__init__(**kwargs) + self.channels = channels + self.corpus_normalize = corpus_normalize + self.causal = causal + self.vae_dim = vae_dim + self.fix_std = fix_std + self.std_dist_type = std_dist_type + + # common parameters + self.conv_norm = conv_norm + self.pad_mode = pad_mode + self.layernorm_eps = layernorm_eps + self.disable_last_norm = disable_last_norm + self.layernorm = layernorm + self.layernorm_elementwise_affine = layernorm_elementwise_affine + self.conv_bias = conv_bias + self.layer_scale_init_value = layer_scale_init_value + self.weight_init_value = weight_init_value + self.mixer_layer = mixer_layer + + # encoder specific parameters + self.encoder_n_filters = encoder_n_filters + self.encoder_ratios = encoder_ratios + self.encoder_depths = encoder_depths + + +class VibeVoiceDiffusionHeadConfig(PretrainedConfig): + model_type = "vibevoice_diffusion_head" + + def __init__( + self, + hidden_size=768, + head_layers=4, + head_ffn_ratio=3.0, + rms_norm_eps=1e-5, + latent_size=64, + speech_vae_dim=None, + prediction_type="v_prediction", + diffusion_type="ddpm", + ddpm_num_steps=1000, + ddpm_num_inference_steps=20, + ddpm_beta_schedule="cosine", + ddpm_batch_mul=4, + **kwargs + ): + self.hidden_size = hidden_size + self.head_layers = head_layers + self.head_ffn_ratio = head_ffn_ratio + self.rms_norm_eps = rms_norm_eps + self.latent_size = latent_size + self.speech_vae_dim = speech_vae_dim + self.prediction_type = prediction_type + self.diffusion_type = diffusion_type + self.ddpm_num_steps = ddpm_num_steps + self.ddpm_num_inference_steps = ddpm_num_inference_steps + self.ddpm_beta_schedule = ddpm_beta_schedule + self.ddpm_batch_mul = ddpm_batch_mul + + super().__init__(**kwargs) + +class VibeVoiceConfig(PretrainedConfig): + model_type = "vibevoice" + is_composition = True + sub_configs = { + "acoustic_tokenizer_config": VibeVoiceAcousticTokenizerConfig, + "semantic_tokenizer_config": VibeVoiceSemanticTokenizerConfig, + "decoder_config": Qwen2Config, + "diffusion_head_config": VibeVoiceDiffusionHeadConfig, + } + # keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `Qwen2` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + + def __init__( + self, + acoustic_tokenizer_config=None, + semantic_tokenizer_config=None, + decoder_config=None, + diffusion_head_config=None, + **kwargs + ): + + # kwargs["_attn_implementation"] = "flash_attention_2" + kwargs["_attn_implementation_autoset"] = False + + if acoustic_tokenizer_config is None: + self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"]() + elif isinstance(acoustic_tokenizer_config, dict): + acoustic_tokenizer_config["model_type"] = "vibevoice_acoustic_tokenizer" + self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"](**acoustic_tokenizer_config) + elif isinstance(acoustic_tokenizer_config, VibeVoiceAcousticTokenizerConfig): + # If an instance of the config class is provided + self.acoustic_tokenizer_config = acoustic_tokenizer_config + + if semantic_tokenizer_config is None: + self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"]() + elif isinstance(semantic_tokenizer_config, dict): + semantic_tokenizer_config["model_type"] = "vibevoice_semantic_tokenizer" + self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"](**semantic_tokenizer_config) + elif isinstance(semantic_tokenizer_config, VibeVoiceSemanticTokenizerConfig): + # If an instance of the config class is provided + self.semantic_tokenizer_config = semantic_tokenizer_config + + if decoder_config is None: + self.decoder_config = self.sub_configs["decoder_config"]() + elif isinstance(decoder_config, dict): + # If a dictionary is provided, instantiate the config class with it + # self.decoder_config = self.sub_configs["decoder_config"](**decoder_config) + if decoder_config.get("model_type", '') == "qwen2": + self.decoder_config = Qwen2Config(**decoder_config) + else: + raise ValueError(f"Unsupported decoder model type: {decoder_config.get('model_type', '')}") + elif isinstance(decoder_config, (Qwen2Config,)): + # If an instance of the config class is provided + self.decoder_config = decoder_config + + if diffusion_head_config is None: + self.diffusion_head_config = self.sub_configs["diffusion_head_config"]() + elif isinstance(diffusion_head_config, dict): + diffusion_head_config["model_type"] = "vibevoice_diffusion_head" + self.diffusion_head_config = self.sub_configs["diffusion_head_config"](**diffusion_head_config) + elif isinstance(diffusion_head_config, VibeVoiceDiffusionHeadConfig): + # If an instance of the config class is provided + self.diffusion_head_config = diffusion_head_config + + # other parameters + self.acoustic_vae_dim = getattr(self.acoustic_tokenizer_config, 'vae_dim', 64) + self.semantic_vae_dim = getattr(self.semantic_tokenizer_config, 'vae_dim', 128) + + # Add attributes required by newer transformers versions from decoder_config + # These are used by GenerationMixin in newer versions + if hasattr(self.decoder_config, 'num_hidden_layers'): + self.num_hidden_layers = self.decoder_config.num_hidden_layers + if hasattr(self.decoder_config, 'vocab_size'): + self.vocab_size = self.decoder_config.vocab_size + if hasattr(self.decoder_config, 'hidden_size'): + self.hidden_size = self.decoder_config.hidden_size + if hasattr(self.decoder_config, 'num_attention_heads'): + self.num_attention_heads = self.decoder_config.num_attention_heads + if hasattr(self.decoder_config, 'num_key_value_heads'): + self.num_key_value_heads = self.decoder_config.num_key_value_heads + if hasattr(self.decoder_config, 'intermediate_size'): + self.intermediate_size = self.decoder_config.intermediate_size + if hasattr(self.decoder_config, 'max_position_embeddings'): + self.max_position_embeddings = self.decoder_config.max_position_embeddings + + super().__init__(**kwargs) + +__all__ = [ + "VibeVoiceAcousticTokenizerConfig", + "VibeVoiceSemanticTokenizerConfig", + "VibeVoiceDiffusionHeadConfig", + "VibeVoiceConfig" +] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/modular/modeling_vibevoice.py b/VibeVoice-ComfyUI/vvembed/modular/modeling_vibevoice.py new file mode 100644 index 0000000000000000000000000000000000000000..54c7c940b678840ac1b6f58628d809694d2446b4 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/modular/modeling_vibevoice.py @@ -0,0 +1,524 @@ +# Original code by Microsoft +# updated by Fabio Sarracino - Enemyx-net + +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple, Union, Callable +from tqdm import tqdm +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.distributed as dist + +from transformers.models.auto import AutoModel, AutoModelForCausalLM + +from transformers.activations import ACT2FN +from transformers.modeling_outputs import CausalLMOutput, BaseModelOutputWithPast, ModelOutput +from transformers.models.llama.modeling_llama import LlamaRMSNorm +from transformers import modeling_utils +from transformers.modeling_utils import PreTrainedModel +from transformers.modeling_flash_attention_utils import FlashAttentionKwargs +from transformers.utils import logging + + +from .modular_vibevoice_tokenizer import VibeVoiceTokenizerStreamingCache, VibeVoiceAcousticTokenizerModel, VibeVoiceSemanticTokenizerModel +from .modular_vibevoice_diffusion_head import VibeVoiceDiffusionHead + +# Import schedule module with robust path handling to avoid conflicts with PyPI 'schedule' package +import sys +import os + +# Get the path to vvembed directory +_current_dir = os.path.dirname(os.path.abspath(__file__)) +_vvembed_dir = os.path.dirname(_current_dir) +_schedule_path = os.path.join(_vvembed_dir, 'schedule') + +# Ensure vvembed is at the front of sys.path to prioritize our schedule module +if _vvembed_dir not in sys.path: + sys.path.insert(0, _vvembed_dir) +elif sys.path.index(_vvembed_dir) > 0: + # Move it to the front if it's not already + sys.path.remove(_vvembed_dir) + sys.path.insert(0, _vvembed_dir) + +# Verify the schedule module exists +if not os.path.exists(_schedule_path): + raise ImportError( + f"Cannot find 'schedule' directory in vvembed. " + f"Expected at: {_schedule_path}" + ) + +# Import with our schedule module prioritized +try: + from schedule.dpm_solver import DPMSolverMultistepScheduler +except ImportError as e: + raise ImportError( + f"Failed to import DPMSolverMultistepScheduler from {_schedule_path}. " + f"There might be a conflict with another Python package. " + f"Original error: {e}" + ) + +from .configuration_vibevoice import VibeVoiceConfig + + +logger = logging.get_logger(__name__) + +if not hasattr(modeling_utils, "ALL_PARALLEL_STYLES") or modeling_utils.ALL_PARALLEL_STYLES is None: + modeling_utils.ALL_PARALLEL_STYLES = ["tp", "none", "colwise", "rowwise"] + +@dataclass +class VibeVoiceCausalLMOutputWithPast(ModelOutput): + loss: Optional[torch.FloatTensor] = None + diffusion_loss: Optional[torch.FloatTensor] = None + speech_token_num: Optional[int] = None + logits: torch.FloatTensor = None + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None + hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None + attentions: Optional[Tuple[torch.FloatTensor, ...]] = None + + +@dataclass +class VibeVoiceGenerationOutput(ModelOutput): + """ + Output type for VibeVoice generation. + + Args: + sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + The generated sequences. + speech_outputs (`List[torch.FloatTensor]`, *optional*): + List of generated speech waveforms or latents for each speech segment. + """ + sequences: torch.LongTensor = None + speech_outputs: Optional[List[torch.FloatTensor]] = None + + +class SpeechConnector(nn.Module): + def __init__(self, input_dim, output_dim): + super().__init__() + self.fc1 = nn.Linear(input_dim, output_dim) + self.norm = LlamaRMSNorm(output_dim, eps=1e-6) + self.fc2 = nn.Linear(output_dim, output_dim) + + def forward(self, features, **kwargs): + x = self.fc1(features) + x = self.norm(x) + x = self.fc2(x) + return x + + +# @auto_docstring +class VibeVoicePreTrainedModel(PreTrainedModel): + config_class = VibeVoiceConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _skip_keys_device_placement = "past_key_values" + _supports_cache_class = True + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_quantized_cache = True + _supports_static_cache = True + _supports_attention_backend = True + + def _init_weights(self, module): + if isinstance(module, VibeVoiceDiffusionHead): + module.initialize_weights() + return + + # Use the language model's initializer_range if available + if hasattr(self.config, 'language_model_config') and hasattr(self.config.language_model_config, 'initializer_range'): + std = self.config.language_model_config.initializer_range + elif hasattr(self.config, 'decoder_config') and hasattr(self.config.decoder_config, 'initializer_range'): + std = self.config.decoder_config.initializer_range + else: + std = 0.02 # Default value + + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.weight.data.fill_(1.0) + module.bias.data.zero_() + +# @auto_docstring +class VibeVoiceModel(VibeVoicePreTrainedModel): + def __init__(self, config): + super().__init__(config) + + if hasattr(config, 'torch_dtype') and config.torch_dtype is not None: + if isinstance(config.torch_dtype, str): + dtype = getattr(torch, config.torch_dtype) + else: + dtype = config.torch_dtype + else: + dtype = torch.float32 + + # Initialize Qwen2 model for language modeling + lm_config = config.decoder_config + self.language_model = AutoModel.from_config(lm_config) + + # Initialize speech components if needed + self.acoustic_tokenizer = AutoModel.from_config(config.acoustic_tokenizer_config).to(dtype) + self.semantic_tokenizer = AutoModel.from_config(config.semantic_tokenizer_config).to(dtype) + + self.acoustic_connector = SpeechConnector(config.acoustic_vae_dim, lm_config.hidden_size).to(dtype) + self.semantic_connector = SpeechConnector(config.semantic_vae_dim, lm_config.hidden_size).to(dtype) + + # Register scaling factors as buffers - use 1D tensors for FSDP compatibility + self.register_buffer('speech_scaling_factor', torch.tensor(float('nan'))) + self.register_buffer('speech_bias_factor', torch.tensor(float('nan'))) + + # Initialize prediction head for speech generation + self.prediction_head = AutoModel.from_config(config.diffusion_head_config).to(dtype) + + # Initialize noise scheduler + self.noise_scheduler = DPMSolverMultistepScheduler( + num_train_timesteps=config.diffusion_head_config.ddpm_num_steps, + beta_schedule=config.diffusion_head_config.ddpm_beta_schedule, + prediction_type=config.diffusion_head_config.prediction_type + ) + + def get_input_embeddings(self): + if hasattr(self.language_model, 'embed_tokens'): + # If the language model has an embed_tokens attribute, return it + return self.language_model.embed_tokens + + for name, attr in self.language_model.fullmap.items(): # parallel by nnscaler, the name is changed + if attr.orig_name == 'embed_tokens.weight': + return getattr(self.language_model, name) + assert False, 'should not arrive here' + + def set_input_embeddings(self, value): + self.language_model.embed_tokens = value + + def set_speech_tokenizers(self, acoustic_tokenizer=None, semantic_tokenizer=None): + """Set the speech tokenizers used for encoding and decoding speech.""" + self.acoustic_tokenizer = acoustic_tokenizer + self.semantic_tokenizer = semantic_tokenizer + + # Reset the encoder to evaluation mode + if self.acoustic_tokenizer is not None: + self.acoustic_tokenizer.eval() + + if self.semantic_tokenizer is not None: + self.semantic_tokenizer.eval() + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Union[Tuple, BaseModelOutputWithPast]: + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # Forward through language model + outputs = self.language_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + **kwargs, + ) + + if not return_dict: + return outputs + + return BaseModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class VibeVoiceForConditionalGeneration(VibeVoicePreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + _tp_plan = {"lm_head": "colwise_rep"} + + def __init__(self, config): + super().__init__(config) + self.model = VibeVoiceModel(config) + self.vocab_size = config.decoder_config.vocab_size + self.lm_head = nn.Linear(config.decoder_config.hidden_size, self.vocab_size, bias=False) + + self.post_init() + + def get_input_embeddings(self): + return self.model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.model.set_input_embeddings(value) + + def get_output_embeddings(self): + return self.lm_head + + def set_decoder(self, decoder): + self.model.language_model = decoder + + def get_decoder(self): + return self.model.language_model + + def tie_weights(self): + """ + Tie the weights between the input embeddings and the output embeddings. + """ + if getattr(self.config.decoder_config, 'tie_word_embeddings', False): + # The standard PreTrainedModel method will handle the tying. + # It typically does a simple parameter object assignment, which is + # CORRECT to do BEFORE FSDP wraps the model. + output_embeddings = self.get_output_embeddings() + input_embeddings = self.get_input_embeddings() + if hasattr(input_embeddings, 'weight'): + output_embeddings.weight = input_embeddings.weight + else: + # maybe returned input_embeddings a tensor directly + output_embeddings.weight = input_embeddings + + if getattr(output_embeddings, "bias", None) is not None: + output_embeddings.bias.data = nn.functional.pad( + output_embeddings.bias.data, + (0, output_embeddings.weight.shape[0] - output_embeddings.bias.shape[0]), + "constant", + 0, + ) + print("✅ Tied input and output embeddings using standard assignment.") + else: + print("ℹ️ tie_word_embeddings is False, not tying weights.") + + # Also, ensure set_output_embeddings is safe, though your implementation looks okay. + # The key is to avoid calling it after accelerator.prepare(). + def set_output_embeddings(self, new_embeddings): + # Your current implementation using data.copy_ is good practice, + # but the best way is to not call this after prepare(). + self.lm_head = new_embeddings + + def forward_speech_features( + self, + speech_tensors=None, + speech_masks=None, + speech_type="audio", + return_unmask=False + ): + if speech_tensors is None: + # Use config to get vae_dim instead of non-existent self.args + vae_dim = self.config.acoustic_tokenizer_config.vae_dim + audio_features = torch.zeros(1, 1, vae_dim).to(self.get_input_embeddings().weight) + connect_features = self.model.acoustic_connector(audio_features) + return audio_features, connect_features + else: + with torch.no_grad(): + if speech_type == "audio": + with torch.no_grad(): + frames = self.model.acoustic_tokenizer.encode(speech_tensors.unsqueeze(1))[0][0] + audio_tokens = frames.sample(self.model.acoustic_tokenizer.std_dist_type)[0] + + elif speech_type == "vae": + # Use config to get vae_dim instead of non-existent self.args + vae_dim = self.config.acoustic_tokenizer_config.vae_dim + speech_mode = speech_tensors.reshape(speech_tensors.size(0), -1, vae_dim) + + # gaussian sample from the speech_mode + batch_size = speech_mode.size(0) + value = self.model.acoustic_tokenizer.fix_std / 0.8 + std = torch.randn(batch_size, dtype=speech_mode.dtype, device=speech_mode.device) * value + std = std.view(-1, *[1] * (speech_mode.dim() - 1)) + audio_tokens = speech_mode + std * torch.randn(speech_mode.shape).to(speech_mode) + else: + raise NotImplementedError(f"Speech type {speech_type} not implemented") + + if torch.isnan(self.model.speech_scaling_factor) or torch.isnan(self.model.speech_bias_factor): + scaling_factor = 1. / audio_tokens[speech_masks].flatten().std() + bias_factor = -audio_tokens[speech_masks].flatten().mean() + + # Only use distributed operations if the process group is initialized + if dist.is_available() and dist.is_initialized(): + dist.all_reduce(scaling_factor, op=dist.ReduceOp.SUM) + dist.all_reduce(bias_factor, op=dist.ReduceOp.SUM) + world_size = dist.get_world_size() + self.model.speech_scaling_factor.copy_(scaling_factor / world_size) + self.model.speech_bias_factor.copy_(bias_factor / world_size) + print(f"Speech scaling factor (distributed): {self.model.speech_scaling_factor}, bias factor: {self.model.speech_bias_factor}", flush=True) + else: + # Single process case + self.model.speech_scaling_factor.copy_(scaling_factor) + self.model.speech_bias_factor.copy_(bias_factor) + print(f"Speech scaling factor (single process): {self.model.speech_scaling_factor}, bias factor: {self.model.speech_bias_factor}", flush=True) + + audio_features = (audio_tokens + self.model.speech_bias_factor) * self.model.speech_scaling_factor + + connect_features = self.model.acoustic_connector(audio_features) + if return_unmask: + return audio_features, connect_features + return audio_features[speech_masks], connect_features[speech_masks] + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = False, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + # New arguments for speech processing and loss calculation + speech_tensors: Optional[torch.FloatTensor] = None, + speech_masks: Optional[torch.BoolTensor] = None, + speeches_loss_input: Optional[torch.FloatTensor] = None, + speech_semantic_tensors: Optional[torch.FloatTensor] = None, + acoustic_input_mask: Optional[torch.BoolTensor] = None, + acoustic_loss_mask: Optional[torch.BoolTensor] = None, + ddpm_batch_mul: int = 1, + **kwargs: Optional[Dict[str, Union[torch.Tensor, str]]], + ) -> Union[Tuple, VibeVoiceCausalLMOutputWithPast]: + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + x = self.get_input_embeddings()(input_ids) + + semantic_speech_all_connect_features = self.model.semantic_connector(speech_semantic_tensors) + if speeches_loss_input is not None: + # only part audio need diffuse + speech_all_features, speech_all_connect_features = self.forward_speech_features( + speech_tensors=speech_tensors.type_as(x) if speech_tensors is not None else None, + speech_masks=speech_masks, + speech_type=kwargs.get("speech_type", "audio"), + return_unmask=True + ) + if speech_tensors is not None: + if semantic_speech_all_connect_features is not None: + x[acoustic_input_mask] = speech_all_connect_features[speech_masks] + semantic_speech_all_connect_features[speech_masks] + else: + x[acoustic_input_mask] = speech_all_connect_features[speech_masks] + speech_features = speech_all_features[speeches_loss_input.unsqueeze(-1) & speech_masks] # only part audio need diffuse + speech_connect_features = speech_all_connect_features[speeches_loss_input.unsqueeze(-1) & speech_masks] + else: + speech_features, speech_connect_features = self.forward_speech_features( + speech_tensors=speech_tensors.type_as(x) if speech_tensors is not None else None, + speech_masks=speech_masks, + speech_type=kwargs.get("speech_type", "audio"), + ) + if speech_tensors is not None: + x[acoustic_input_mask] = speech_connect_features + + outputs = self.model( + input_ids=None, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=x, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=False, + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs.last_hidden_state + logits = self.lm_head(hidden_states) + # logits = logits.float() + + loss = None + if labels is not None: + # The custom CE loss with masking is calculated in the training script. + # We leave the standard loss calculation here as None. + pass + + # --- Diffusion Loss Calculation --- + diffusion_loss = None + # This block is executed only if we are in a context that involves speech. + if speech_tensors is not None and acoustic_loss_mask.sum().item() > 0: + condition_features = hidden_states[acoustic_loss_mask] + + speech_len, latent_size = speech_features.shape + + noise = torch.randn( + (speech_len * ddpm_batch_mul, latent_size), + device=hidden_states.device, + dtype=hidden_states.dtype + ) + + timesteps = torch.multinomial( + torch.ones(self.config.diffusion_head_config.ddpm_num_steps), + speech_len * ddpm_batch_mul, + replacement=True, + ).to(hidden_states.device) + + speech_features_repeated = speech_features.repeat_interleave(ddpm_batch_mul, dim=0) + condition_features_repeated = condition_features.repeat_interleave(ddpm_batch_mul, dim=0) + + noisy_speech_features = self.model.noise_scheduler.add_noise( + speech_features_repeated, noise, timesteps + ) + + model_output = self.model.prediction_head( + noisy_speech_features, + timesteps.type_as(x), + condition_features_repeated + ) + + prediction_type = self.config.diffusion_head_config.prediction_type + if prediction_type == "epsilon": + target_for_loss = noise + elif prediction_type == "v_prediction": + target_for_loss = self.model.noise_scheduler.get_velocity( + speech_features_repeated, noise, timesteps + ) + else: + raise NotImplementedError(f"Prediction type {prediction_type} not implemented") + + diffusion_loss = F.mse_loss(model_output.float(), target_for_loss.float(), reduction='sum') + if latent_size > 0 and ddpm_batch_mul > 0: + diffusion_loss = diffusion_loss / latent_size / ddpm_batch_mul + else: + diffusion_loss = torch.tensor(0.0, device=diffusion_loss.device) + + else: + # Dummy loss for DDP to work when there are no speech samples in a batch, + # but we are in a speech context. + diffusion_loss = sum(p.sum() for p in self.model.prediction_head.parameters()) * 0.0 + diffusion_loss += sum(p.sum() for p in self.model.acoustic_connector.parameters()) * 0.0 + diffusion_loss += sum(p.sum() for p in self.model.semantic_connector.parameters()) * 0.0 + # --- End Diffusion Loss Calculation --- + + if not return_dict: + output = (logits, speech_len) + outputs.to_tuple()[1:] + return (loss, diffusion_loss) + output + + return VibeVoiceCausalLMOutputWithPast( + loss=loss, + diffusion_loss=diffusion_loss, + speech_token_num=speech_len if speech_tensors is not None else 0, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + +AutoModel.register(VibeVoiceConfig, VibeVoiceModel) +AutoModelForCausalLM.register(VibeVoiceConfig, VibeVoiceForConditionalGeneration) + +__all__ = [ + "VibeVoiceModel", + "VibeVoicePreTrainedModel", + "VibeVoiceForConditionalGeneration", + "VibeVoiceCausalLMOutputWithPast", + "VibeVoiceGenerationOutput", +] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/modular/modeling_vibevoice_inference.py b/VibeVoice-ComfyUI/vvembed/modular/modeling_vibevoice_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..7961d4b0bf0220f82bf9ebf4a83ebefade2b8662 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/modular/modeling_vibevoice_inference.py @@ -0,0 +1,871 @@ +# Original code by Microsoft +# updated by Fabio Sarracino - Enemyx-net + +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple, Union, Callable +from tqdm import tqdm +import torch +import torch.nn as nn + +from transformers.models.auto import AutoModel, AutoModelForCausalLM + +from transformers.generation import GenerationMixin, GenerationConfig, LogitsProcessor, LogitsProcessorList, StoppingCriteriaList +from transformers.modeling_outputs import BaseModelOutputWithPast, ModelOutput +from transformers import modeling_utils +from transformers.modeling_utils import PreTrainedModel +from transformers.modeling_flash_attention_utils import FlashAttentionKwargs +from transformers.utils import logging + + +# from .modular_vibevoice_tokenizer import VibeVoiceTokenizerStreamingCache, VibeVoiceAcousticTokenizerModel, VibeVoiceSemanticTokenizerModel +from .modular_vibevoice_tokenizer import VibeVoiceTokenizerStreamingCache, VibeVoiceTokenizerEncoderOutput +from .modular_vibevoice_diffusion_head import VibeVoiceDiffusionHead + +# Import schedule module with robust path handling to avoid conflicts with PyPI 'schedule' package +import sys +import os + +# Get the path to vvembed directory +_current_dir = os.path.dirname(os.path.abspath(__file__)) +_vvembed_dir = os.path.dirname(_current_dir) +_schedule_path = os.path.join(_vvembed_dir, 'schedule') + +# Ensure vvembed is at the front of sys.path to prioritize our schedule module +if _vvembed_dir not in sys.path: + sys.path.insert(0, _vvembed_dir) +elif sys.path.index(_vvembed_dir) > 0: + # Move it to the front if it's not already + sys.path.remove(_vvembed_dir) + sys.path.insert(0, _vvembed_dir) + +# Verify the schedule module exists +if not os.path.exists(_schedule_path): + raise ImportError( + f"Cannot find 'schedule' directory in vvembed. " + f"Expected at: {_schedule_path}" + ) + +# Import with our schedule module prioritized +try: + from schedule.dpm_solver import DPMSolverMultistepScheduler +except ImportError as e: + raise ImportError( + f"Failed to import DPMSolverMultistepScheduler from {_schedule_path}. " + f"There might be a conflict with another Python package. " + f"Original error: {e}" + ) + +from .configuration_vibevoice import VibeVoiceConfig + +from .modular_vibevoice_text_tokenizer import VibeVoiceTextTokenizer, VibeVoiceTextTokenizerFast + +from .modeling_vibevoice import VibeVoiceModel, VibeVoicePreTrainedModel +from .streamer import AudioStreamer, AsyncAudioStreamer + +logger = logging.get_logger(__name__) + +if not hasattr(modeling_utils, "ALL_PARALLEL_STYLES") or modeling_utils.ALL_PARALLEL_STYLES is None: + modeling_utils.ALL_PARALLEL_STYLES = ["tp", "none", "colwise", "rowwise"] + +@dataclass +class VibeVoiceCausalLMOutputWithPast(BaseModelOutputWithPast): + logits: Optional[torch.FloatTensor] = None + +@dataclass +class VibeVoiceGenerationOutput(ModelOutput): + """ + Output type for VibeVoice generation. + + Args: + sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + The generated sequences. + speech_outputs (`List[torch.FloatTensor]`, *optional*): + List of generated speech waveforms or latents for each speech segment. + """ + sequences: torch.LongTensor = None + speech_outputs: Optional[List[torch.FloatTensor]] = None + reach_max_step_sample: Optional[torch.BoolTensor] = None + +class VibeVoiceTokenConstraintProcessor(LogitsProcessor): + """Constrains token generation to only valid tokens during speech generation.""" + + def __init__(self, valid_token_ids: List[int], device: torch.device = None): + self.valid_token_ids = torch.tensor(valid_token_ids, dtype=torch.long, device=device) + + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # Create a mask for valid tokens + mask = torch.full_like(scores, float('-inf')) + mask[:, self.valid_token_ids] = 0 + + # Apply mask to scores + scores = scores + mask + return scores + +def access_cache_safely(cache, layer_idx): + """Access cache tensors safely across different transformers versions + + This function handles the different DynamicCache structures across transformers versions: + - Old versions (< 4.36): cache.key_cache, cache.value_cache + - Intermediate versions: cache._keys, cache._values + - New versions (4.36+): Various new structures + + Returns (k_cache, v_cache) or (None, None) if cache structure is incompatible + """ + try: + # Method 1: Old versions (< 4.36) + if hasattr(cache, 'key_cache') and hasattr(cache, 'value_cache'): + if layer_idx < len(cache.key_cache): + return cache.key_cache[layer_idx], cache.value_cache[layer_idx] + + # Method 2: Private attributes (some intermediate versions) + if hasattr(cache, '_keys') and hasattr(cache, '_values'): + if layer_idx < len(cache._keys): + return cache._keys[layer_idx], cache._values[layer_idx] + + # Method 3: New versions with get_seq_length or similar + # Some versions store as list of tuples + if isinstance(cache, (list, tuple)) and len(cache) > layer_idx: + layer_cache = cache[layer_idx] + if isinstance(layer_cache, (list, tuple)) and len(layer_cache) >= 2: + return layer_cache[0], layer_cache[1] + elif hasattr(layer_cache, 'key_states') and hasattr(layer_cache, 'value_states'): + return layer_cache.key_states, layer_cache.value_states + + # Method 4: Check if cache has a different structure entirely + # Some very new versions might not expose cache directly + if hasattr(cache, 'to_legacy_tuple'): + # Convert to legacy format if possible + legacy = cache.to_legacy_tuple() + if legacy and layer_idx < len(legacy): + return legacy[layer_idx][0], legacy[layer_idx][1] + + except (AttributeError, IndexError, TypeError) as e: + # Log the issue but don't fail + logger.debug(f"Could not access cache at layer {layer_idx}: {e}") + + # Return None if we can't access the cache safely + return None, None + +def get_num_layers_from_cache(cache): + """Get the number of layers in the cache structure""" + try: + if hasattr(cache, 'key_cache'): + return len(cache.key_cache) + elif hasattr(cache, '_keys'): + return len(cache._keys) + elif isinstance(cache, (list, tuple)): + return len(cache) + elif hasattr(cache, 'num_layers'): + return cache.num_layers + # Default fallback - most models have 32 or fewer layers + return 32 + except: + return 32 + +class VibeVoiceForConditionalGenerationInference(VibeVoicePreTrainedModel, GenerationMixin): + _tied_weights_keys = ["lm_head.weight"] + _tp_plan = {"lm_head": "colwise_rep"} + + def __init__(self, config): + super().__init__(config) + + # Initialize the base model + self.model = VibeVoiceModel(config) + + # LM head for text generation + self.lm_head = nn.Linear(config.decoder_config.hidden_size, config.decoder_config.vocab_size, bias=False) + + # inference configuration + self.ddpm_inference_steps = config.diffusion_head_config.ddpm_num_inference_steps + + # Initialize weights and apply final processing + self.post_init() + + @property + def noise_scheduler(self): + return self.model.noise_scheduler + + @property + def prediction_head(self): + return self.model.prediction_head + + @property + def speech_scaling_factor(self): + return self.model.speech_scaling_factor + + @property + def speech_bias_factor(self): + return self.model.speech_bias_factor + + @property + def acoustic_tokenizer(self): + return self.model.acoustic_tokenizer + + @property + def semantic_tokenizer(self): + return self.model.semantic_tokenizer + + @property + def acoustic_connector(self): + return self.model.acoustic_connector + + @property + def semantic_connector(self): + return self.model.semantic_connector + + def tie_weights(self): + """ + Tie the weights between the input embeddings and the output embeddings. + """ + # Tie lm_head.weight to language_model.embed_tokens.weight + if not getattr(self.config, 'tie_word_embeddings', False): + return + + if hasattr(self, 'lm_head') and hasattr(self.model.language_model, 'embed_tokens'): + self.lm_head.weight = self.model.language_model.embed_tokens.weight + + def get_input_embeddings(self): + return self.model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.model.set_input_embeddings(value) + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_speech_tokenizers(self, acoustic_tokenizer=None, semantic_tokenizer=None): + """Set the speech tokenizers used for encoding and decoding speech.""" + self.model.set_speech_tokenizers(acoustic_tokenizer, semantic_tokenizer) + + def set_ddpm_inference_steps(self, num_steps=None): + self.ddpm_inference_steps = num_steps or self.config.diffusion_head_config.ddpm_num_inference_steps + + def _process_speech_inputs(self, speech_tensors, speech_masks, speech_type="audio"): + """Process speech inputs through tokenizers and connectors.""" + with torch.no_grad(): + if speech_type == "audio": + # Encode audio to acoustic latents + encoder_output = self.model.acoustic_tokenizer.encode(speech_tensors.unsqueeze(1)) + acoustic_latents = encoder_output.sample(dist_type=self.model.acoustic_tokenizer.std_dist_type)[0] + + # Apply scaling and bias + acoustic_features = (acoustic_latents + self.model.speech_bias_factor.to(acoustic_latents.device)) * self.model.speech_scaling_factor.to(acoustic_latents.device) + + # Connect to language model space + acoustic_connected = self.model.acoustic_connector(acoustic_features)[speech_masks.cpu()] + + return acoustic_features, acoustic_connected + elif speech_type == "pt": + encoder_output = VibeVoiceTokenizerEncoderOutput(mean=speech_tensors, std=self.acoustic_tokenizer.config.fix_std) + acoustic_latents = encoder_output.sample(dist_type=self.model.acoustic_tokenizer.std_dist_type)[0] + + # Apply scaling and bias + acoustic_features = (acoustic_latents + self.model.speech_bias_factor.to(acoustic_latents.device)) * self.model.speech_scaling_factor.to(acoustic_latents.device) + + # Connect to language model space + acoustic_connected = self.model.acoustic_connector(acoustic_features)[speech_masks.cpu()] + + return acoustic_features, acoustic_connected + else: + raise NotImplementedError(f"Speech type {speech_type} not implemented") + + # @can_return_tuple + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + speech_tensors: Optional[torch.FloatTensor] = None, + speech_masks: Optional[torch.BoolTensor] = None, + speech_input_mask: Optional[torch.BoolTensor] = None, + logits_to_keep: Union[int, slice] = 0, + **kwargs, + ) -> Union[Tuple, VibeVoiceCausalLMOutputWithPast]: + """ + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + speech_tensors (`torch.FloatTensor`, *optional*): + Input speech waveforms for voice cloning or speech understanding. + speech_masks (`torch.BoolTensor`, *optional*): + Masks indicating valid speech frames. + speech_input_mask (`torch.BoolTensor`, *optional*): + Positions in the input sequence where speech embeddings should be inserted. + + Returns: + `VibeVoiceCausalLMOutputWithPast` or tuple + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # Get embeddings + if inputs_embeds is None: + inputs_embeds = self.model.get_input_embeddings()(input_ids) + + # Process speech inputs if provided + if speech_tensors is not None and speech_masks is not None: + acoustic_features, speech_embeds = self._process_speech_inputs(speech_tensors.to(self.dtype), speech_masks) + if speech_input_mask is not None: + inputs_embeds[speech_input_mask] = speech_embeds + + outputs = self.model( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs[0] if not return_dict else outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + if labels is not None: + raise NotImplementedError("Loss computation is not implemented in this version.") + + return VibeVoiceCausalLMOutputWithPast( + logits=logits, + past_key_values=outputs.past_key_values, + last_hidden_state=hidden_states, + attentions=outputs.attentions, + ) + + def _build_generate_config_model_kwargs(self, generation_config, inputs, tokenizer, return_processors=False, **kwargs): + if generation_config is None: + generation_config = GenerationConfig( + bos_token_id=tokenizer.bos_token_id, + eos_token_id=tokenizer.eos_token_id, + pad_token_id = tokenizer.pad_token_id + ) + else: + generation_config = GenerationConfig( + **generation_config, + bos_token_id=tokenizer.bos_token_id, + eos_token_id=tokenizer.eos_token_id, + pad_token_id = tokenizer.pad_token_id + ) + + generation_config, model_kwargs = self._prepare_generation_config( + generation_config, + True, + speech_start_id=tokenizer.speech_start_id, + speech_end_id=tokenizer.speech_end_id, + speech_diffusion_id=tokenizer.speech_diffusion_id, + **kwargs + ) + generation_config.speech_start_id = tokenizer.speech_start_id + generation_config.speech_end_id = tokenizer.speech_end_id + generation_config.speech_diffusion_id = tokenizer.speech_diffusion_id + + inputs_tensor, model_input_name, model_kwargs = self._prepare_model_inputs(inputs, generation_config.bos_token_id, model_kwargs) + batch_size = inputs_tensor.shape[0] + device = self.device + + self._prepare_special_tokens(generation_config, True, device=device) + generation_config.use_cache = True + model_kwargs["use_cache"] = generation_config.use_cache + input_ids = inputs_tensor.to(self.device) + + input_ids_length = input_ids.shape[1] + has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None + has_default_min_length = kwargs.get("min_length") is None and generation_config.min_length is not None + generation_config = self._prepare_generated_length( + generation_config=generation_config, + has_default_max_length=has_default_max_length, + has_default_min_length=has_default_min_length, + model_input_name=model_input_name, + inputs_tensor=inputs_tensor, + input_ids_length=input_ids_length, + ) + + max_cache_length = generation_config.max_length - 1 + + # Fix for transformers compatibility: detect number of parameters accepted + import inspect + try: + sig = inspect.signature(self._prepare_cache_for_generation) + num_params = len(sig.parameters) + + # Newer transformers expects 6 parameters (without 'device') + # Older transformers expects 7 parameters (with 'device') + if num_params == 6 or 'device' not in sig.parameters: + # New signature: (self, generation_config, model_kwargs, assistant_model, batch_size, max_cache_length) + self._prepare_cache_for_generation(generation_config, model_kwargs, None, batch_size, max_cache_length) + else: + # Old signature: (self, generation_config, model_kwargs, assistant_model, batch_size, max_cache_length, device) + self._prepare_cache_for_generation(generation_config, model_kwargs, None, batch_size, max_cache_length, device) + except Exception as e: + # Fallback: try both signatures + try: + # Try new signature first (6 params) + self._prepare_cache_for_generation(generation_config, model_kwargs, None, batch_size, max_cache_length) + except TypeError: + # Fall back to old signature (7 params) + self._prepare_cache_for_generation(generation_config, model_kwargs, None, batch_size, max_cache_length, device) + + model_kwargs['cache_position'] = torch.arange(input_ids_length, device=device, dtype=torch.long) + for k, v in model_kwargs.items(): + if isinstance(v, torch.Tensor): + model_kwargs[k] = v.to(device=device) + + if return_processors: + logits_processor = self._get_logits_processor( + generation_config=generation_config, + input_ids_seq_length=input_ids_length, + encoder_input_ids=inputs_tensor, + prefix_allowed_tokens_fn=None, + logits_processor=LogitsProcessorList(), + device=inputs_tensor.device, + model_kwargs=model_kwargs, + ) + + stopping_criteria = self._get_stopping_criteria(generation_config=generation_config, stopping_criteria=StoppingCriteriaList()) + + return generation_config, model_kwargs, input_ids, logits_processor, stopping_criteria + else: + return generation_config, model_kwargs, input_ids + + @torch.no_grad() + def generate( + self, + inputs: Optional[torch.Tensor] = None, + generation_config: Optional[GenerationConfig] = None, + logits_processor: Optional[LogitsProcessorList] = None, + stopping_criteria: Optional[StoppingCriteriaList] = None, + prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None, + synced_gpus: Optional[bool] = None, + assistant_model: Optional["PreTrainedModel"] = None, + audio_streamer: Optional[Union[AudioStreamer, AsyncAudioStreamer]] = None, + negative_prompt_ids: Optional[torch.Tensor] = None, + negative_prompt_attention_mask: Optional[torch.Tensor] = None, + speech_tensors: Optional[torch.FloatTensor] = None, + speech_masks: Optional[torch.BoolTensor] = None, + speech_input_mask: Optional[torch.BoolTensor] = None, + return_speech: bool = True, + cfg_scale: float = 1.0, + stop_check_fn: Optional[Callable[[], bool]] = None, + **kwargs, + ) -> Union[torch.LongTensor, VibeVoiceGenerationOutput]: + """ + Generates sequences of token ids and optionally speech outputs. + + Args: + All standard generation arguments from GenerationMixin + negative_prompt_ids: Negative prompt for CFG in speech generation + negative_prompt_attention_mask: Attention mask for negative prompt + speech_tensors: Input speech for voice cloning + speech_masks: Masks for speech tensors + speech_input_mask: Positions to insert speech embeddings + return_speech: Whether to decode and return speech outputs + cfg_scale: CFG scale for speech generation + stop_check_fn: Optional callable that returns True if generation should stop + + Returns: + Generated token sequences and optionally speech outputs + """ + # 1. Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call + tokenizer = kwargs.pop("tokenizer", None) # Pull this out first, we only use it for stopping criteria + parsed_scripts = kwargs.pop("parsed_scripts", None) + all_speakers_list = kwargs.pop("all_speakers_list", None) + max_length_times = kwargs.pop("max_length_times", 2) + + if kwargs.get('max_new_tokens', None) is None: + kwargs['max_new_tokens'] = self.config.decoder_config.max_position_embeddings - kwargs['input_ids'].shape[-1] + + generation_config, model_kwargs, input_ids, logits_processor, stopping_criteria = self._build_generate_config_model_kwargs( + generation_config, inputs, tokenizer, return_processors=True, **kwargs + ) + + negative_kwargs = { + 'input_ids': torch.full((kwargs['input_ids'].shape[0], 1), tokenizer.speech_start_id, dtype=torch.long, device=kwargs['input_ids'].device), + 'attention_mask': torch.ones((kwargs['input_ids'].shape[0], 1), dtype=torch.long, device=kwargs['input_ids'].device), + 'max_new_tokens': kwargs.get('max_new_tokens', 100) + } + negative_generation_config, negative_model_kwargs, negative_input_ids = self._build_generate_config_model_kwargs( + None, None, tokenizer, return_processors=False, **negative_kwargs + ) + + acoustic_cache = VibeVoiceTokenizerStreamingCache() + semantic_cache = VibeVoiceTokenizerStreamingCache() + + batch_size = input_ids.shape[0] + device = input_ids.device + finished_tags = torch.zeros(batch_size, dtype=torch.bool, device=device) + correct_cnt = torch.zeros(batch_size, dtype=torch.long, device=device) + is_prefill = True + inputs_embeds = None + verbose = kwargs.get("verbose", False) + + # Initialize audio chunks storage for each sample + audio_chunks = [[] for _ in range(batch_size)] + + initial_length = input_ids.shape[-1] + initial_length_per_sample = model_kwargs['attention_mask'].sum(dim=-1) + + # Define all valid tokens that can be generated + valid_tokens = [ + generation_config.speech_start_id, + generation_config.speech_end_id, + generation_config.speech_diffusion_id, + generation_config.eos_token_id + ] + # Add bos_token_id if it exists + if hasattr(generation_config, 'bos_token_id') and generation_config.bos_token_id is not None: + valid_tokens.append(generation_config.bos_token_id) + + # Add custom processor to constrain token generation + token_constraint_processor = VibeVoiceTokenConstraintProcessor(valid_tokens, device=device) + if logits_processor is None: + logits_processor = LogitsProcessorList() + logits_processor.append(token_constraint_processor) + + max_steps = min(generation_config.max_length - initial_length, int(max_length_times * initial_length)) + max_step_per_sample = torch.min(generation_config.max_length - initial_length_per_sample, (max_length_times * initial_length_per_sample).long()) + reach_max_step_sample = torch.zeros(batch_size, dtype=torch.bool, device=device) + + # Create progress iterator if verbose + if kwargs.get("show_progress_bar", True): + progress_bar = tqdm(range(max_steps), desc="Generating", leave=False) + else: + progress_bar = range(max_steps) + + for step in progress_bar: + # Check for external stop signal + if stop_check_fn is not None and stop_check_fn(): + if verbose: + print(f"Generation stopped externally at step {step + 1}") + # End the audio streamer if it exists + if audio_streamer is not None: + audio_streamer.end() + break + + # Check if audio_streamer has been ended (stopped externally) + if audio_streamer is not None and hasattr(audio_streamer, 'finished_flags'): + if any(audio_streamer.finished_flags): + if verbose: + print(f"Audio generation stopped externally at step {step + 1}") + break + + if finished_tags.all(): + if hasattr(progress_bar, 'set_description'): + progress_bar.set_description("Generation complete") + break + + if input_ids.shape[-1] >= generation_config.max_length: + print(f"Reached maximum generation length {generation_config.max_length}, stopped it.") + reached_samples = torch.arange(batch_size, device=device)[~finished_tags] + if reached_samples.numel() > 0: + reach_max_step_sample[reached_samples] = True + break + + # Update progress bar description with active samples + if hasattr(progress_bar, 'set_description'): + active_samples = (~finished_tags).sum().item() + progress_bar.set_description(f"Generating (active: {active_samples}/{batch_size})") + + model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs) + if is_prefill: + # we process the speech inputs only during the first generation step + prefill_inputs = { + "speech_tensors": speech_tensors.to(device=device), + "speech_masks": speech_masks.to(device), + "speech_input_mask": speech_input_mask.to(device), + } + is_prefill = False + else: + _ = model_inputs.pop('inputs_embeds', None) + prefill_inputs = {'inputs_embeds': inputs_embeds} + + # Forward pass through the model + outputs = self( + **model_inputs, **prefill_inputs, logits_to_keep=1, return_dict=True, output_attentions=False, output_hidden_states=False, + ) + model_kwargs = self._update_model_kwargs_for_generation( + outputs, model_kwargs, is_encoder_decoder=False, + ) + + # Get logits and apply logits processor + next_token_logits = outputs.logits[:, -1, :].to(copy=True, dtype=torch.float32, device=input_ids.device) + # next_token_logits = outputs.logits[:, -1, :].to(copy=True, device=input_ids.device) + next_token_scores = logits_processor(input_ids, next_token_logits) + + # token selection + if generation_config.do_sample: + probs = nn.functional.softmax(next_token_scores, dim=-1) + # TODO (joao): this OP throws "skipping cudagraphs due to ['incompatible ops']", find solution + next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1) + else: + next_tokens = torch.argmax(next_token_scores, dim=-1) + + next_tokens[finished_tags] = generation_config.eos_token_id + input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + + if not kwargs.get('refresh_negative', True): + negative_model_inputs = self.prepare_inputs_for_generation(negative_input_ids, **negative_model_kwargs) + # Forward negative pass through the model + if negative_model_inputs['inputs_embeds'] is None and inputs_embeds is not None: + negative_model_inputs['inputs_embeds'] = inputs_embeds + negative_model_inputs['input_ids'] = None + + negative_outputs = self( + **negative_model_inputs, logits_to_keep=0, return_dict=True, output_attentions=False, output_hidden_states=False, + ) + negative_model_kwargs = self._update_model_kwargs_for_generation( + negative_outputs, negative_model_kwargs, is_encoder_decoder=False, + ) + negative_input_ids = torch.cat([negative_input_ids, next_tokens[:, None]], dim=-1) + + # reached end of generation + if (next_tokens == generation_config.eos_token_id).any(): + eos_indices = (next_tokens == generation_config.eos_token_id).nonzero(as_tuple=False).squeeze(1) + # Only print for samples that are newly finished (not already marked as finished) + new_eos_indices = eos_indices[~finished_tags[eos_indices]] + if new_eos_indices.numel() > 0: + finished_tags[new_eos_indices] = True + if verbose: + print(f"Samples {new_eos_indices.tolist()} reached EOS token at step {step + 1}.", flush=True) + if audio_streamer is not None: + audio_streamer.end(new_eos_indices) + + # Check if any sample reached its maximum generation length + max_length_reached = step >= max_step_per_sample + new_max_length_indices = torch.nonzero(max_length_reached & ~finished_tags, as_tuple=False).squeeze(1) + if new_max_length_indices.numel() > 0: + finished_tags[new_max_length_indices] = True + reach_max_step_sample[new_max_length_indices] = True + if verbose: + print(f"Samples {new_max_length_indices.tolist()} reached max generation length at step {step + 1}.", flush=True) + if audio_streamer is not None: + audio_streamer.end(new_max_length_indices) + + # speech_end + diffusion_end_indices = (next_tokens == generation_config.speech_end_id).nonzero(as_tuple=False).squeeze(1) + if diffusion_end_indices.numel() > 0: + # Clear tokenizer caches for samples that reached speech end + acoustic_cache.set_to_zero(diffusion_end_indices) + semantic_cache.set_to_zero(diffusion_end_indices) + + # speech_begin + diffusion_start_indices = torch.arange(batch_size, device=device)[~finished_tags & (next_tokens == generation_config.speech_start_id)] + if diffusion_start_indices.numel() > 0 and kwargs.get('refresh_negative', True): + # update attention mask + for i, sample_idx in enumerate(diffusion_start_indices.tolist()): + negative_model_kwargs['attention_mask'][sample_idx, :] = 0 + negative_model_kwargs['attention_mask'][sample_idx, -1] = 1 + # update past key values - using safe cache access + cache = negative_model_kwargs['past_key_values'] + num_layers = get_num_layers_from_cache(cache) + cache_update_failed = False + + for layer_idx in range(num_layers): + k_cache, v_cache = access_cache_safely(cache, layer_idx) + if k_cache is None or v_cache is None: + # Cache structure not compatible, skip optimization + logger.debug(f"Cache optimization skipped at layer {layer_idx} - incompatible structure") + cache_update_failed = True + break + + # Process each non-diffusion sample + for sample_idx in diffusion_start_indices.tolist(): + try: + # Shift cache for this sample + k_cache[sample_idx, :, -1, :] = k_cache[sample_idx, :, 0, :].clone() + v_cache[sample_idx, :, -1, :] = v_cache[sample_idx, :, 0, :].clone() + except (IndexError, RuntimeError) as e: + logger.debug(f"Cache update failed for sample {sample_idx}: {e}") + cache_update_failed = True + break + + if cache_update_failed: + break + # update negative_input_ids + for sample_idx in diffusion_start_indices.tolist(): + negative_input_ids[sample_idx, -1] = generation_config.speech_start_id + + # Prepare inputs_embeds for next iteration + # Initialize with default embeddings for all tokens + next_inputs_embeds = self.model.get_input_embeddings()(next_tokens).unsqueeze(1) # [batch_size, 1, hidden_size] + + # forward diffusion + # Diffusion indices are those that are not finished and not special tokens + diffusion_indices = torch.arange(batch_size, device=device)[~finished_tags & (next_tokens == generation_config.speech_diffusion_id)] + + if diffusion_indices.numel() > 0: + if kwargs.get('refresh_negative', True): + negative_model_inputs = self.prepare_inputs_for_generation(negative_input_ids, **negative_model_kwargs) + # Forward negative pass through the model + if negative_model_inputs['inputs_embeds'] is None and inputs_embeds is not None: + negative_model_inputs['inputs_embeds'] = inputs_embeds + negative_model_inputs['input_ids'] = None + + negative_outputs = self( + **negative_model_inputs, logits_to_keep=0, return_dict=True, output_attentions=False, output_hidden_states=False, + ) + negative_model_kwargs = self._update_model_kwargs_for_generation( + negative_outputs, negative_model_kwargs, is_encoder_decoder=False, + ) + negative_input_ids = torch.cat([negative_input_ids, next_tokens[:, None]], dim=-1) + # correct the non-diffusion indices + # we forward all samples' negative outputs even if + # they are not in diffusion mode to keep the cache consistent + # So we need to correct the kv cache of non-diffusion samples + non_diffusion_mask = ~finished_tags & (next_tokens != generation_config.speech_diffusion_id) + if non_diffusion_mask.any(): + non_diffusion_indices = torch.arange(batch_size, device=device)[non_diffusion_mask] + start_indices = correct_cnt[non_diffusion_indices] + + # 1. Update attention_mask - need to handle each sample separately + seq_len = negative_model_kwargs['attention_mask'].shape[1] + for i, (sample_idx, start_idx) in enumerate(zip(non_diffusion_indices.tolist(), start_indices.tolist())): + # Shift the attention mask for this sample + if start_idx + 1 < seq_len - 1: + negative_model_kwargs['attention_mask'][sample_idx, start_idx+1:] = \ + negative_model_kwargs['attention_mask'][sample_idx, start_idx:-1].clone() + negative_model_kwargs['attention_mask'][sample_idx, start_idx] = 0 + + # 2. Update past_key_values - using safe cache access + cache = negative_model_kwargs['past_key_values'] + num_layers = get_num_layers_from_cache(cache) + cache_update_failed = False + + for layer_idx in range(num_layers): + k_cache, v_cache = access_cache_safely(cache, layer_idx) + if k_cache is None or v_cache is None: + # Cache structure not compatible, skip optimization + logger.debug(f"Cache optimization skipped at layer {layer_idx} - incompatible structure") + cache_update_failed = True + break + + # Process each non-diffusion sample + for sample_idx, start_idx in zip(non_diffusion_indices.tolist(), start_indices.tolist()): + try: + if start_idx + 1 < k_cache.shape[2] - 1: + # Shift cache for this sample + k_cache[sample_idx, :, start_idx+1:, :] = k_cache[sample_idx, :, start_idx:-1, :].clone() + v_cache[sample_idx, :, start_idx+1:, :] = v_cache[sample_idx, :, start_idx:-1, :].clone() + except (IndexError, RuntimeError) as e: + logger.debug(f"Cache update failed for sample {sample_idx}: {e}") + cache_update_failed = True + break + + if cache_update_failed: + break + + # 3. Update negative_input_ids + for sample_idx, start_idx in zip(non_diffusion_indices.tolist(), start_indices.tolist()): + if start_idx + 1 < negative_input_ids.shape[1] - 1: + negative_input_ids[sample_idx, start_idx+1:] = \ + negative_input_ids[sample_idx, start_idx:-1].clone() + + correct_cnt[non_diffusion_indices] += 1 + + positive_condition = outputs.last_hidden_state[diffusion_indices, -1, :] + negative_condition = negative_outputs.last_hidden_state[diffusion_indices, -1, :] + + speech_latent = self.sample_speech_tokens( + positive_condition, + negative_condition, + cfg_scale=cfg_scale, + ).unsqueeze(1) + + # Decode acoustic latent to audio using acoustic streaming cache + scaled_latent = speech_latent / self.model.speech_scaling_factor.to(speech_latent.device) - self.model.speech_bias_factor.to(speech_latent.device) + audio_chunk = self.model.acoustic_tokenizer.decode( + scaled_latent.to(self.model.acoustic_tokenizer.device), + cache=acoustic_cache, # Use acoustic-specific cache + sample_indices=diffusion_indices.to(self.model.acoustic_tokenizer.device), + use_cache=True, + debug=False + ) + + # Store audio chunks for each sample + for i, sample_idx in enumerate(diffusion_indices): + idx = sample_idx.item() + # Only append audio chunk if the sample is not finished + if not finished_tags[idx]: + audio_chunks[idx].append(audio_chunk[i]) + + # Add streaming support here + if audio_streamer is not None: + # Stream the audio chunks immediately + audio_streamer.put(audio_chunk, diffusion_indices) + + # Encode audio to semantic features using semantic streaming cache + semantic_features = self.model.semantic_tokenizer.encode( + audio_chunk, + cache=semantic_cache, # Use semantic-specific cache + sample_indices=diffusion_indices, + use_cache=True, + debug=False + ).mean # semantic tokenizer has no VAE. + + # Combine acoustic and semantic features for next input + acoustic_embed = self.model.acoustic_connector(speech_latent) + semantic_embed = self.model.semantic_connector(semantic_features) + diffusion_embeds = acoustic_embed + semantic_embed + + # Update embeddings for diffusion indices + next_inputs_embeds[diffusion_indices] = diffusion_embeds + + # Set inputs_embeds for next iteration + inputs_embeds = next_inputs_embeds + + if audio_streamer is not None: + audio_streamer.end() + + # Concatenate audio chunks for each sample + final_audio_outputs = [] + for sample_chunks in audio_chunks: + if sample_chunks: + # Concatenate all chunks along the time dimension (assumed to be the last dimension) + concatenated_audio = torch.cat(sample_chunks, dim=-1) + final_audio_outputs.append(concatenated_audio) + else: + # If no audio was generated for this sample, append None + final_audio_outputs.append(None) + + return VibeVoiceGenerationOutput( + sequences=input_ids, + speech_outputs=final_audio_outputs if return_speech else None, + reach_max_step_sample=reach_max_step_sample, + ) + + @torch.no_grad() + def sample_speech_tokens(self, condition, neg_condition, cfg_scale=3.0): + self.model.noise_scheduler.set_timesteps(self.ddpm_inference_steps) + condition = torch.cat([condition, neg_condition], dim=0).to(self.model.prediction_head.device) + speech = torch.randn(condition.shape[0], self.config.acoustic_vae_dim).to(condition) + for t in self.model.noise_scheduler.timesteps: + half = speech[: len(speech) // 2] + combined = torch.cat([half, half], dim=0) + eps = self.model.prediction_head(combined, t.repeat(combined.shape[0]).to(combined), condition=condition) + cond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0) + half_eps = uncond_eps + cfg_scale * (cond_eps - uncond_eps) + eps = torch.cat([half_eps, half_eps], dim=0) + speech = self.model.noise_scheduler.step(eps, t, speech).prev_sample + return speech[: len(speech) // 2] + + +AutoModelForCausalLM.register(VibeVoiceConfig, VibeVoiceForConditionalGenerationInference) + +__all__ = [ + "VibeVoiceForConditionalGenerationInference", +] diff --git a/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_diffusion_head.py b/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_diffusion_head.py new file mode 100644 index 0000000000000000000000000000000000000000..7ac231ebd03a90edfaa65e4c367867a20d20d3c6 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_diffusion_head.py @@ -0,0 +1,287 @@ +import math +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers.models.auto import AutoModel +from transformers.modeling_utils import PreTrainedModel +# from transformers.modeling_layers import GradientCheckpointingLayer +from transformers.activations import ACT2FN +from transformers.utils import logging + +from .configuration_vibevoice import VibeVoiceDiffusionHeadConfig + + +logger = logging.get_logger(__name__) + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6, elementwise_affine=True, memory_efficient=False): + super().__init__() + self.dim = dim + self.eps = eps + self.elementwise_affine = elementwise_affine + if self.elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim)) + else: + self.register_parameter('weight', None) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()).type_as(x) + if self.weight is not None: + output = output * self.weight + return output + + def extra_repr(self) -> str: + return f'dim={self.dim}, eps={self.eps}, elementwise_affine={self.elementwise_affine}' + +def modulate(x, shift, scale): + """Apply modulation to input tensor.""" + return x * (1 + scale) + shift + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + + Args: + hidden_size (`int`): Size of the output embedding + frequency_embedding_size (`int`, optional): Size of the intermediate frequency embedding + """ + def __init__(self, hidden_size, frequency_embedding_size=256): + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=False), + # nn.SiLU(), + ACT2FN['silu'], + nn.Linear(hidden_size, hidden_size, bias=False), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + + Args: + t (`torch.Tensor`): A 1-D Tensor of N indices, one per batch element. + These may be fractional. + dim (`int`): The dimension of the output. + max_period (`int`, optional): Controls the minimum frequency of the embeddings. + + Returns: + `torch.Tensor`: An [N, D] Tensor of positional embeddings. + """ + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half + ).to(t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding.to(t.dtype) + + def forward(self, t): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + t_emb = self.mlp(t_freq) + return t_emb + + +class FeedForwardNetwork(nn.Module): + """ + Standard feed-forward network with SwiGLU activation. + + Args: + embed_dim (`int`): Input dimension + ffn_dim (`int`): Hidden dimension + """ + def __init__( + self, + embed_dim, + ffn_dim, + ): + super().__init__() + self.embed_dim = embed_dim + self.gate_proj = nn.Linear(self.embed_dim, ffn_dim, bias=False) + self.up_proj = nn.Linear(self.embed_dim, ffn_dim, bias=False) + self.down_proj = nn.Linear(ffn_dim, self.embed_dim, bias=False) + self.act_fn = ACT2FN['silu'] # Using SiLU as the activation function + + def forward(self, x): + gate = self.gate_proj(x) + up = self.up_proj(x) + + # SwiGLU activation + # gate = F.silu(gate) + gate = self.act_fn(gate) + return self.down_proj(gate * up) + + +class HeadLayer(nn.Module): + """ + A layer in the diffusion head. + + Args: + embed_dim (`int`): Input dimension + ffn_dim (`int`): Hidden dimension + cond_dim (`int`): Condition embedding dimension + norm_eps (`float`, optional): Epsilon for normalization + """ + def __init__( + self, + embed_dim, + ffn_dim, + cond_dim, + norm_eps=1e-5, + ): + super().__init__() + self.embed_dim = embed_dim + self.cond_dim = cond_dim + self.ffn_dim = ffn_dim + self.ffn = FeedForwardNetwork( + self.embed_dim, + self.ffn_dim, + ) + self.norm = RMSNorm(self.embed_dim, eps=norm_eps) + self.adaLN_modulation = nn.Sequential( + # nn.SiLU(), + ACT2FN['silu'], + nn.Linear(cond_dim, 3 * self.embed_dim, bias=False) + ) + + def forward(self, x, c): + shift_ffn, scale_ffn, gate_ffn = self.adaLN_modulation(c).chunk(3, dim=-1) + x = x + gate_ffn * self.ffn(modulate(self.norm(x), shift_ffn, scale_ffn)) + return x + + +class FinalLayer(nn.Module): + """ + Final layer in the diffusion head. + + Args: + hidden_size (`int`): Input dimension + output_size (`int`): Output dimension + cond_size (`int`): Condition embedding dimension + norm_eps (`float`, optional): Epsilon for normalization + """ + def __init__(self, hidden_size, output_size, cond_size, norm_eps=1e-5): + super().__init__() + self.norm_final = RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=False) + self.linear = nn.Linear(hidden_size, output_size, bias=False) + self.adaLN_modulation = nn.Sequential( + # nn.SiLU(), + ACT2FN['silu'], + nn.Linear(cond_size, 2 * hidden_size, bias=False) + ) + + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1) + x = modulate(self.norm_final(x), shift, scale) + x = self.linear(x) + return x + + +class VibeVoiceDiffusionHead(PreTrainedModel): + """ + Diffusion head model for vibevoice. + + Args: + config (`VibeVoiceDiffusionHeadConfig`): Model configuration + latent_size (`int`, optional): Size of the latent space. If not provided, uses `config.latent_size`. + """ + config_class = VibeVoiceDiffusionHeadConfig + supports_gradient_checkpointing = True + _supports_flash_attn_2 = True + _supports_sdpa = True + + def __init__( + self, + config, + ): + super().__init__(config) + self.config = config + self.cond_dim = config.hidden_size + latent_size = config.latent_size + + self.noisy_images_proj = nn.Linear(latent_size, config.hidden_size, bias=False) + self.cond_proj = nn.Linear(config.hidden_size, self.cond_dim, bias=False) + self.t_embedder = TimestepEmbedder(self.cond_dim) + + ffn_dim = int(config.hidden_size * config.head_ffn_ratio) + + # Create the intermediate layers + self.layers = nn.ModuleList([ + HeadLayer( + embed_dim=config.hidden_size, + ffn_dim=ffn_dim, + cond_dim=self.cond_dim, + norm_eps=config.rms_norm_eps + ) + for _ in range(config.head_layers) + ]) + + # Final layer for output + self.final_layer = FinalLayer( + hidden_size=config.hidden_size, + output_size=latent_size, + cond_size=self.cond_dim, + norm_eps=config.rms_norm_eps + ) + + self.initialize_weights() + + def initialize_weights(self): + """Initialize the weights of the model.""" + # Initialize timestep embedder + nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02) + + # Zero-out adaLN modulation layers + for layer in self.layers: + nn.init.constant_(layer.adaLN_modulation[-1].weight, 0) + + # Zero-out output layers + nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0) + nn.init.constant_(self.final_layer.linear.weight, 0) + + def forward( + self, + noisy_images, + timesteps, + condition, + ): + """ + Forward pass of the prediction head. + + Args: + noisy_images (`torch.Tensor`): Noisy images/latents to denoise + timesteps (`torch.Tensor`): Timesteps for diffusion + condition (`torch.Tensor`): Conditioning information + + Returns: + `torch.Tensor`: The predicted noise/velocity + """ + x = self.noisy_images_proj(noisy_images) + t = self.t_embedder(timesteps) + condition = self.cond_proj(condition) + c = condition + t + + for layer in self.layers: + x = layer(x, c) + + x = self.final_layer(x, c) + return x + + +AutoModel.register(VibeVoiceDiffusionHeadConfig, VibeVoiceDiffusionHead) + +__all__ = [ + "VibeVoiceDiffusionHead", +] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_text_tokenizer.py b/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_text_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..444c77cc718d3993c291f3c97bf9f07052f681b1 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_text_tokenizer.py @@ -0,0 +1,214 @@ +"""Tokenization classes for vibevoice.""" + +from typing import List, Optional, Union + +from transformers.utils import logging +from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer +from transformers.models.qwen2.tokenization_qwen2_fast import Qwen2TokenizerFast + +logger = logging.get_logger(__name__) + + +class VibeVoiceTextTokenizer(Qwen2Tokenizer): + """ + Construct a VibeVoice tokenizer. Based on the Qwen2 tokenizer with additional special tokens for speech. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + merges_file (`str`): + Path to the merges file. + errors (`str`, *optional*, defaults to `"replace"`): + Paradigm to follow when decoding bytes to UTF-8. + unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`): + The unknown token. + bos_token (`str`, *optional*): + The beginning of sequence token. Not used for vibevoice. + eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): + The end of sequence token. + pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`): + The token used for padding. + add_special_tokens (`bool`, *optional*, defaults to `True`): + Whether or not to add special tokens when encoding. + """ + + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + merges_file, + errors="replace", + unk_token="<|endoftext|>", + bos_token=None, + eos_token="<|endoftext|>", + pad_token="<|endoftext|>", + add_prefix_space=False, + add_special_tokens=True, + **kwargs, + ): + super().__init__( + vocab_file=vocab_file, + merges_file=merges_file, + errors=errors, + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + add_prefix_space=add_prefix_space, + add_special_tokens=add_special_tokens, + **kwargs, + ) + + # Add VibeVoice-specific special tokens + self._add_vibevoice_special_tokens() + + def _add_vibevoice_special_tokens(self): + """Add VibeVoice-specific special tokens.""" + special_tokens = { + "additional_special_tokens": [ + "<|vision_start|>", # Speech start (reusing vision tokens) + "<|vision_end|>", # Speech end + "<|vision_pad|>", # Speech diffusion pad + ] + } + num_added = self.add_special_tokens(special_tokens) + + # Cache special token IDs + self._speech_start_id = self.convert_tokens_to_ids("<|vision_start|>") + self._speech_end_id = self.convert_tokens_to_ids("<|vision_end|>") + self._speech_diffusion_id = self.convert_tokens_to_ids("<|vision_pad|>") + + self._eos_id = self.convert_tokens_to_ids('<|endoftext|>') + + return num_added + + @property + def eos_id(self) -> int: + """Id of the end of sequence token.""" + return self._eos_id + + @property + def speech_start_id(self) -> int: + """Id of the speech start token.""" + return self._speech_start_id + + @property + def speech_end_id(self) -> int: + """Id of the speech end token.""" + return self._speech_end_id + + @property + def speech_diffusion_id(self) -> int: + """Id of the speech diffusion token.""" + return self._speech_diffusion_id + + @property + def pad_id(self) -> int: + """Id used for padding (returns -100 for loss masking).""" + return -100 + + +class VibeVoiceTextTokenizerFast(Qwen2TokenizerFast): + """ + Construct a "fast" VibeVoice tokenizer (backed by HuggingFace's *tokenizers* library). + Based on the Qwen2 tokenizer with additional special tokens for speech. + + Args: + vocab_file (`str`, *optional*): + Path to the vocabulary file. + merges_file (`str`, *optional*): + Path to the merges file. + tokenizer_file (`str`, *optional*): + Path to [tokenizers](https://github.com/huggingface/tokenizers) file. + unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`): + The unknown token. + bos_token (`str`, *optional*): + The beginning of sequence token. Not used for vibevoice. + eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): + The end of sequence token. + pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`): + The token used for padding. + """ + + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file=None, + merges_file=None, + tokenizer_file=None, + unk_token="<|endoftext|>", + bos_token=None, + eos_token="<|endoftext|>", + pad_token="<|endoftext|>", + add_prefix_space=False, + **kwargs, + ): + super().__init__( + vocab_file=vocab_file, + merges_file=merges_file, + tokenizer_file=tokenizer_file, + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + add_prefix_space=add_prefix_space, + **kwargs, + ) + + # Add VibeVoice-specific special tokens + self._add_vibevoice_special_tokens() + + def _add_vibevoice_special_tokens(self): + """Add VibeVoice-specific special tokens.""" + special_tokens = { + "additional_special_tokens": [ + "<|vision_start|>", # Speech start (reusing vision tokens) + "<|vision_end|>", # Speech end + "<|vision_pad|>", # Speech diffusion pad + ] + } + num_added = self.add_special_tokens(special_tokens) + + # Cache special token IDs + self._speech_start_id = self.convert_tokens_to_ids("<|vision_start|>") + self._speech_end_id = self.convert_tokens_to_ids("<|vision_end|>") + self._speech_diffusion_id = self.convert_tokens_to_ids("<|vision_pad|>") + + # self._eos_id = self.convert_tokens_to_ids('<|endoftext|>') + self._eos_id = self.eos_token_id # qwen2 / qwen3 + self._pad_id = self.convert_tokens_to_ids('<|image_pad|>') + + return num_added + + @property + def eos_id(self) -> int: + """Id of the end of sequence token.""" + return self._eos_id + + @property + def speech_start_id(self) -> int: + """Id of the speech start token.""" + return self._speech_start_id + + @property + def speech_end_id(self) -> int: + """Id of the speech end token.""" + return self._speech_end_id + + @property + def speech_diffusion_id(self) -> int: + """Id of the speech diffusion token.""" + return self._speech_diffusion_id + + @property + def pad_id(self) -> int: + """Id used for padding (returns -100 for loss masking).""" + return self._pad_id + + +__all__ = [ + "VibeVoiceTextTokenizer", + "VibeVoiceTextTokenizerFast", +] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_tokenizer.py b/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..ff727f30f268b84051c46639b966990ef6f02a67 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/modular/modular_vibevoice_tokenizer.py @@ -0,0 +1,1195 @@ +import math +import typing as tp +from functools import partial +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple, Union +import copy + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers.models.auto import AutoModel + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging +from transformers.modeling_utils import PreTrainedModel +from transformers.activations import ACT2FN + +from .configuration_vibevoice import VibeVoiceAcousticTokenizerConfig, VibeVoiceSemanticTokenizerConfig + +logger = logging.get_logger(__name__) + +import os +# Try to import APEX FusedRMSNorm +try: + from apex.normalization.fused_layer_norm import fused_rms_norm_affine + APEX_AVAILABLE = True + logger.info("APEX FusedRMSNorm is available and will be used for optimization") + if int(os.getenv("OPTIMIZE_FOR_SPEED", "0")) == 0: + APEX_AVAILABLE = False + logger.warning("APEX FusedRMSNorm is disabled by environment variable OPTIMIZE_FOR_SPEED=0") +except ImportError: + APEX_AVAILABLE = False + logger.warning("APEX FusedRMSNorm not available, using native implementation") +# APEX_AVAILABLE=False + +# Normalization modules +class ConvLayerNorm(nn.LayerNorm): + """ + Convolution-friendly LayerNorm that moves channels to last dimensions + before running the normalization and moves them back to original position right after. + """ + def __init__(self, normalized_shape: tp.Union[int, tp.List[int], torch.Size], **kwargs): + super().__init__(normalized_shape, **kwargs) + + def forward(self, x): + x = x.transpose(1, 2) # b ... t -> b t ... + x = nn.functional.layer_norm(x.float(), self.normalized_shape, self.weight.float(), self.bias.float(), self.eps).type_as(x) + x = x.transpose(1, 2) # b t ... -> b ... t + return x + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-5, elementwise_affine=True, weight_shape=None): + super().__init__() + self.dim = dim + self.eps = eps + self.elementwise_affine = elementwise_affine + if self.elementwise_affine: + weight_shape = (dim,) if weight_shape is None else weight_shape + self.weight = nn.Parameter(torch.ones(weight_shape)) + else: + self.register_parameter('weight', None) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()).type_as(x) + if self.weight is not None: + output = output * self.weight + return output + + def extra_repr(self) -> str: + return f'dim={self.dim}, eps={self.eps}, elementwise_affine={self.elementwise_affine}' + +class ConvRMSNorm(RMSNorm): + def __init__(self, dim: int, eps: float = 1e-5, elementwise_affine=True, weight_shape=None): + super().__init__(dim, eps, elementwise_affine, weight_shape) + + def forward(self, x): + x = x.transpose(1, 2) # b ... t -> b t ... + if (not APEX_AVAILABLE) or (not self.elementwise_affine): + # Fallback to native implementation + output = self._norm(x.float()).type_as(x) + if self.weight is not None: + output = output * self.weight + else: + output = fused_rms_norm_affine(x, self.weight, self.weight.shape, self.eps) + output = output.transpose(1, 2) # b t ... -> b ... t + return output + +# Convolutional layers and utilities +CONV_NORMALIZATIONS = frozenset(['none', 'weight_norm', 'spectral_norm', + 'time_layer_norm', 'layer_norm', 'time_group_norm']) + + +def apply_parametrization_norm(module: nn.Module, norm: str = 'none') -> nn.Module: + assert norm in CONV_NORMALIZATIONS + if norm == 'weight_norm': + return nn.utils.weight_norm(module) + elif norm == 'spectral_norm': + return nn.utils.spectral_norm(module) + else: + # We already check was in CONV_NORMALIZATION, so any other choice + # doesn't need reparametrization. + return module + + +def get_norm_module(module: nn.Module, causal: bool = False, norm: str = 'none', **norm_kwargs) -> nn.Module: + """Return the proper normalization module. If causal is True, this will ensure the returned + module is causal, or return an error if the normalization doesn't support causal evaluation. + """ + assert norm in CONV_NORMALIZATIONS + if norm == 'layer_norm': + assert isinstance(module, nn.modules.conv._ConvNd) + return ConvLayerNorm(module.out_channels, **norm_kwargs) + elif norm == 'time_group_norm': + if causal: + raise ValueError("GroupNorm doesn't support causal evaluation.") + assert isinstance(module, nn.modules.conv._ConvNd) + return nn.GroupNorm(1, module.out_channels, **norm_kwargs) + else: + return nn.Identity() + + +def get_extra_padding_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int, + padding_total: int = 0) -> int: + """Calculate extra padding needed for convolution to have the same output length""" + length = x.shape[-1] + n_frames = (length - kernel_size + padding_total) / stride + 1 + ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total) + return ideal_length - length + + +def pad1d(x: torch.Tensor, paddings: tp.Tuple[int, int], mode: str = 'zero', value: float = 0.): + """Pad 1D input with handling for small inputs in reflect mode""" + length = x.shape[-1] + padding_left, padding_right = paddings + assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right) + if mode == 'reflect': + max_pad = max(padding_left, padding_right) + extra_pad = 0 + if length <= max_pad: + extra_pad = max_pad - length + 1 + x = F.pad(x, (0, extra_pad)) + padded = F.pad(x, paddings, mode, value) + end = padded.shape[-1] - extra_pad + return padded[..., :end] + else: + return F.pad(x, paddings, mode, value) + + +def unpad1d(x: torch.Tensor, paddings: tp.Tuple[int, int]): + """Remove padding from x, handling properly zero padding. Only for 1d!""" + padding_left, padding_right = paddings + assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right) + assert (padding_left + padding_right) <= x.shape[-1] + end = x.shape[-1] - padding_right + return x[..., padding_left: end] + + +class NormConv1d(nn.Module): + """Wrapper around Conv1d and normalization applied to this conv""" + def __init__(self, *args, causal: bool = False, norm: str = 'none', + norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs): + super().__init__() + self.conv = apply_parametrization_norm(nn.Conv1d(*args, **kwargs), norm) + self.norm = get_norm_module(self.conv, causal, norm, **norm_kwargs) + self.norm_type = norm + + def forward(self, x): + x = self.conv(x) + x = self.norm(x) + return x + + +class NormConvTranspose1d(nn.Module): + """Wrapper around ConvTranspose1d and normalization applied to this conv""" + def __init__(self, *args, causal: bool = False, norm: str = 'none', + norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs): + super().__init__() + self.convtr = apply_parametrization_norm(nn.ConvTranspose1d(*args, **kwargs), norm) + self.norm = get_norm_module(self.convtr, causal, norm, **norm_kwargs) + self.norm_type = norm + + def forward(self, x): + x = self.convtr(x) + x = self.norm(x) + return x + + +class VibeVoiceTokenizerStreamingCache: + """Cache for streaming convolution, similar to KV cache in attention""" + def __init__(self): + self.cache = {} # Dict mapping (layer_id, sample_idx) to state tensor + + def get(self, layer_id: str, sample_indices: torch.Tensor) -> Optional[torch.Tensor]: + """Get cached states for given layer and sample indices""" + states = [] + max_length = 0 + + # First pass: collect states and find max length + for idx in sample_indices.tolist(): + key = (layer_id, idx) + if key not in self.cache: + return None # If any sample is missing, return None + state = self.cache[key] + states.append(state) + max_length = max(max_length, state.shape[-1]) + + # Second pass: pad states to max length if needed + if len(states) > 0 and states[0].dim() >= 2: + padded_states = [] + for state in states: + if state.shape[-1] < max_length: + # Pad on the time dimension (last dimension) + pad_size = max_length - state.shape[-1] + # Pad with zeros on the LEFT to align the most recent samples + padded_state = F.pad(state, (pad_size, 0), mode='constant', value=0) + padded_states.append(padded_state) + else: + padded_states.append(state) + return torch.stack(padded_states, dim=0) + else: + return torch.stack(states, dim=0) + + def set(self, layer_id: str, sample_indices: torch.Tensor, states: torch.Tensor): + """Set cached states for given layer and sample indices""" + for i, idx in enumerate(sample_indices.tolist()): + key = (layer_id, idx) + self.cache[key] = states[i].detach() + + def set_to_zero(self, sample_indices: torch.Tensor): + """Set all cached states to zero for given sample indices""" + for key in list(self.cache.keys()): + layer_id, sample_idx = key + if sample_idx in sample_indices.tolist(): + # Create zero tensor with same shape and dtype as cached tensor + cached_tensor = self.cache[key] + self.cache[key] = torch.zeros_like(cached_tensor) + + def clear(self, layer_id: Optional[str] = None, sample_indices: Optional[torch.Tensor] = None): + """Clear cache for specific layer/samples or everything""" + if layer_id is None and sample_indices is None: + self.cache.clear() + elif layer_id is not None and sample_indices is None: + # Clear all samples for a specific layer + keys_to_remove = [k for k in self.cache.keys() if k[0] == layer_id] + for k in keys_to_remove: + del self.cache[k] + elif layer_id is not None and sample_indices is not None: + # Clear specific samples for a specific layer + for idx in sample_indices.tolist(): + key = (layer_id, idx) + self.cache.pop(key, None) + +class SConv1d(nn.Module): + """Conv1d with built-in handling of asymmetric or causal padding and normalization.""" + def __init__(self, in_channels: int, out_channels: int, + kernel_size: int, stride: int = 1, dilation: int = 1, + groups: int = 1, bias: bool = True, causal: bool = False, + norm: str = 'none', norm_kwargs: tp.Dict[str, tp.Any] = {}, + pad_mode: str = 'reflect'): + super().__init__() + self.conv = NormConv1d(in_channels, out_channels, kernel_size, stride, + dilation=dilation, groups=groups, bias=bias, causal=causal, + norm=norm, norm_kwargs=norm_kwargs) + self.causal = causal + self.pad_mode = pad_mode + + # Store configuration + self.kernel_size = kernel_size + self.dilation = dilation + self.stride = stride + self.in_channels = in_channels + self.out_channels = out_channels + + # For causal convolution, we need to maintain kernel_size - 1 samples as context + # need to check use which context_size is more suitable + # self.context_size = (kernel_size - 1) * dilation + self.context_size = (kernel_size - 1) * dilation - (stride - 1) + + # For non-streaming mode, calculate padding + self.padding_total = (kernel_size - 1) * dilation - (stride - 1) + + # Create a unique layer ID for cache management + self._layer_id = None + + @property + def layer_id(self): + if self._layer_id is None: + self._layer_id = f"sconv1d_{id(self)}" + return self._layer_id + + def forward(self, x: torch.Tensor, + cache: Optional[VibeVoiceTokenizerStreamingCache] = None, + sample_indices: Optional[torch.Tensor] = None, + use_cache: bool = False, + debug: bool = False) -> torch.Tensor: + """ + Forward pass with optional streaming support via cache. + + Args: + x: Input tensor [batch_size, channels, time] + cache: VibeVoiceTokenizerStreamingCache object for maintaining states + sample_indices: Indices identifying each sample for cache management + use_cache: Whether to use cached states for streaming + debug: Whether to print debug information + + Returns: + Output tensor + """ + B, C, T = x.shape + + # Non-streaming mode + if not use_cache or cache is None: + return self._forward_non_streaming(x, debug=debug) + + # Streaming mode + assert self.causal, "Streaming mode is only supported for causal convolutions" + assert sample_indices is not None, "sample_indices must be provided for streaming mode" + assert len(sample_indices) == B, "sample_indices must match batch size" + + return self._forward_streaming(x, cache, sample_indices, debug) + + def _forward_streaming(self, x: torch.Tensor, + cache: VibeVoiceTokenizerStreamingCache, + sample_indices: torch.Tensor, + debug: bool = False) -> torch.Tensor: + """Streaming forward pass with cache operations kept separate from compiled code""" + B, C, T = x.shape + + # Cache operations (not compiled) + cached_states = cache.get(self.layer_id, sample_indices) + + if cached_states is None: + # First chunk - initialize with zeros for context + if self.context_size > 0: + cached_states = torch.zeros(B, C, self.context_size, device=x.device, dtype=x.dtype) + if debug: + print(f"[DEBUG] Initialized cache with shape: {cached_states.shape}, context_size={self.context_size}") + else: + cached_states = torch.zeros(B, C, 0, device=x.device, dtype=x.dtype) + if debug: + print(f"[DEBUG] No context needed (kernel_size=stride)") + + # Concatenate cached states with input + if cached_states.shape[2] > 0: + input_with_context = torch.cat([cached_states, x], dim=2) + else: + input_with_context = x + + if debug: + print(f"[DEBUG] Input shape: {x.shape}, Cache shape: {cached_states.shape}, Combined: {input_with_context.shape}") + + # Apply convolution directly - no extra padding in streaming mode + # The conv layer will handle its own padding internally + output = self.conv(input_with_context) + + if debug: + print(f"[DEBUG] Output shape: {output.shape}") + + # Update cache for next chunk + if self.context_size > 0: + # Calculate how many samples to keep + total_input_length = input_with_context.shape[2] + + # Keep the last context_size samples + if total_input_length >= self.context_size: + new_cache_start = total_input_length - self.context_size + new_cache = input_with_context[:, :, new_cache_start:] + else: + # If we have less than context_size samples, keep everything + new_cache = input_with_context + + if debug: + print(f"[DEBUG] New cache shape: {new_cache.shape}") + + cache.set(self.layer_id, sample_indices, new_cache) + + return output + + def _forward_non_streaming(self, x: torch.Tensor, debug: bool = False) -> torch.Tensor: + """Standard forward pass without streaming""" + B, C, T = x.shape + kernel_size = self.kernel_size + stride = self.stride + dilation = self.dilation + padding_total = self.padding_total + + # Compute extra padding for stride alignment + extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total) + + if debug: + print(f"[DEBUG NON-STREAMING] Input shape: {x.shape}, padding_total={padding_total}, extra_padding={extra_padding}") + + if self.causal: + # Left padding for causal + if self.pad_mode == 'constant': + x = pad1d(x, (padding_total, extra_padding), mode=self.pad_mode, value=0) + else: + x = pad1d(x, (padding_total, extra_padding), mode=self.pad_mode) + else: + # Symmetric padding for non-causal + padding_right = padding_total // 2 + padding_left = padding_total - padding_right + x = pad1d(x, (padding_left, padding_right + extra_padding), mode=self.pad_mode) + + if debug: + print(f"[DEBUG NON-STREAMING] After padding: {x.shape}") + + output = self.conv(x) + + if debug: + print(f"[DEBUG NON-STREAMING] Output shape: {output.shape}") + + return output + + +class SConvTranspose1d(nn.Module): + """ConvTranspose1d with built-in handling of asymmetric or causal padding and normalization.""" + def __init__(self, in_channels: int, out_channels: int, + kernel_size: int, stride: int = 1, causal: bool = False, + norm: str = 'none', trim_right_ratio: float = 1., + norm_kwargs: tp.Dict[str, tp.Any] = {}, bias: bool = True): + super().__init__() + self.convtr = NormConvTranspose1d(in_channels, out_channels, kernel_size, stride, + causal=causal, norm=norm, norm_kwargs=norm_kwargs, bias=bias) + self.causal = causal + self.trim_right_ratio = trim_right_ratio + assert self.causal or self.trim_right_ratio == 1., \ + "`trim_right_ratio` != 1.0 only makes sense for causal convolutions" + assert self.trim_right_ratio >= 0. and self.trim_right_ratio <= 1. + + # Store configuration + self.kernel_size = kernel_size + self.stride = stride + self.in_channels = in_channels + self.out_channels = out_channels + + # For transposed convolution, padding calculation is different + self.padding_total = kernel_size - stride + + # For streaming, we need to keep track of input history + # Transposed conv needs to see multiple input samples to produce correct output + self.context_size = kernel_size - 1 + + # Create a unique layer ID for cache management + self._layer_id = None + + @property + def layer_id(self): + if self._layer_id is None: + self._layer_id = f"sconvtr1d_{id(self)}" + return self._layer_id + + def forward(self, x: torch.Tensor, + cache: Optional[VibeVoiceTokenizerStreamingCache] = None, + sample_indices: Optional[torch.Tensor] = None, + use_cache: bool = False, + debug: bool = False) -> torch.Tensor: + """ + Forward pass with optional streaming support via cache. + """ + B, C, T = x.shape + + # Non-streaming mode + if not use_cache or cache is None: + return self._forward_non_streaming(x, debug=debug) + + # Streaming mode + assert sample_indices is not None, "sample_indices must be provided for streaming mode" + assert len(sample_indices) == B, "sample_indices must match batch size" + + return self._forward_streaming(x, cache, sample_indices, debug) + + def _forward_streaming(self, x: torch.Tensor, + cache: VibeVoiceTokenizerStreamingCache, + sample_indices: torch.Tensor, + debug: bool = False) -> torch.Tensor: + """Streaming forward pass with cache operations kept separate from compiled code""" + B, C, T = x.shape + + # Cache operations (not compiled) + cached_input = cache.get(self.layer_id, sample_indices) + + if cached_input is None: + # First chunk - no history yet + cached_input = torch.zeros(B, C, 0, device=x.device, dtype=x.dtype) + if debug: + print(f"[DEBUG] Initialized empty cache for transposed conv") + + # Concatenate cached input with new input + full_input = torch.cat([cached_input, x], dim=2) + + if debug: + print(f"[DEBUG] Input shape: {x.shape}, Cache shape: {cached_input.shape}, Combined: {full_input.shape}") + + # First chunk or debug mode - use uncompiled version + full_output = self.convtr(full_input) + + if debug: + print(f"[DEBUG] Full transposed conv output shape: {full_output.shape}") + + # Calculate padding to remove + if self.causal: + padding_right = math.ceil(self.padding_total * self.trim_right_ratio) + padding_left = self.padding_total - padding_right + else: + padding_right = self.padding_total // 2 + padding_left = self.padding_total - padding_right + + # Remove padding + if padding_left + padding_right > 0: + full_output = unpad1d(full_output, (padding_left, padding_right)) + + if debug: + print(f"[DEBUG] After unpadding: {full_output.shape}") + + # Determine which part of the output corresponds to the new input + if cached_input.shape[2] == 0: + # First chunk - return all output + output = full_output + else: + # Subsequent chunks - return only the new output + expected_new_output = T * self.stride + + # Take the last expected_new_output samples + if full_output.shape[2] >= expected_new_output: + output = full_output[:, :, -expected_new_output:] + else: + output = full_output + + if debug: + print(f"[DEBUG] Final streaming output shape: {output.shape}") + + # Update cache + if full_input.shape[2] > self.context_size: + new_cache = full_input[:, :, -self.context_size:] + else: + new_cache = full_input + + if debug: + print(f"[DEBUG] New cache shape: {new_cache.shape}") + + cache.set(self.layer_id, sample_indices, new_cache) + + return output + + def _forward_non_streaming(self, x: torch.Tensor, debug: bool = False) -> torch.Tensor: + """Standard forward pass without streaming""" + if debug: + print(f"[DEBUG NON-STREAMING] Input shape: {x.shape}") + + # Apply transposed convolution + y = self.convtr(x) + + if debug: + print(f"[DEBUG NON-STREAMING] After transposed conv: {y.shape}") + + # Calculate and remove padding + if self.causal: + padding_right = math.ceil(self.padding_total * self.trim_right_ratio) + padding_left = self.padding_total - padding_right + else: + padding_right = self.padding_total // 2 + padding_left = self.padding_total - padding_right + + if padding_left + padding_right > 0: + y = unpad1d(y, (padding_left, padding_right)) + + if debug: + print(f"[DEBUG NON-STREAMING] Final output shape: {y.shape}") + + return y + +# FFN +class FFN(nn.Module): + def __init__( + self, + embed_dim, + ffn_dim, + bias=False, + ): + super().__init__() + self.embed_dim = embed_dim + self.linear1 = nn.Linear(self.embed_dim, ffn_dim, bias=bias) + self.gelu = ACT2FN["gelu"] + self.linear2 = nn.Linear(ffn_dim, self.embed_dim, bias=bias) + + def forward(self, x): + x = self.linear1(x) + x = self.gelu(x) + x = self.linear2(x) + return x + + +class Convlayer(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + dilation=1, + groups=1, + bias=True, + pad_mode='zeros', + norm='weight_norm', + causal=True, + ): + super().__init__() + self.conv = SConv1d(in_channels, out_channels, kernel_size, stride=stride, dilation=dilation, + groups=groups, bias=bias, pad_mode=pad_mode, norm=norm, causal=causal) + + def forward(self, x): + return self.conv(x) + +class Block1D(nn.Module): + def __init__(self, dim, kernel_size=7, drop_path=0., mixer_layer='conv', + layer_scale_init_value=1e-6, **kwargs): + super().__init__() + + if kwargs.get('layernorm', 'LN') == 'LN': + self.norm = ConvLayerNorm(dim, eps=kwargs.get('eps', 1e-6)) + self.ffn_norm = ConvLayerNorm(dim, eps=kwargs.get('eps', 1e-6)) + elif kwargs.get('layernorm', 'RMSNorm') == 'RMSNorm': + self.norm = ConvRMSNorm(dim, eps=kwargs.get('eps', 1e-6)) + self.ffn_norm = ConvRMSNorm(dim, eps=kwargs.get('eps', 1e-6)) + + if mixer_layer == 'conv': + self.mixer = Convlayer(dim, dim, groups=kwargs.get('groups', 1), + kernel_size=kernel_size, + pad_mode=kwargs.get('pad_mode', 'reflect'), + norm=kwargs.get('norm', 'none'), + causal=kwargs.get('causal', True), + bias=kwargs.get('bias', True), + ) + elif mixer_layer == 'depthwise_conv': + self.mixer = Convlayer(dim, dim, groups=dim, + kernel_size=kernel_size, + pad_mode=kwargs.get('pad_mode', 'reflect'), + norm=kwargs.get('norm', 'none'), + causal=kwargs.get('causal', True), + bias=kwargs.get('bias', True), + ) + else: + raise ValueError(f"Unsupported mixer layer: {mixer_layer}") + + self.ffn = FFN( + dim, + kwargs.get('ffn_expansion', 4) * dim, + bias=kwargs.get('bias', False), + ) + self.drop_path = nn.Identity() if drop_path <= 0. else nn.modules.DropPath(drop_path) + + if layer_scale_init_value > 0: + self.gamma = nn.Parameter(layer_scale_init_value * torch.ones((dim)), requires_grad=True) + self.ffn_gamma = nn.Parameter(layer_scale_init_value * torch.ones((dim)), requires_grad=True) + else: + self.gamma = None + self.ffn_gamma = None + + def forward(self, x): + # mixer + residual = x + x = self.norm(x) + x = self.mixer(x) + if self.gamma is not None: + x = x * self.gamma.unsqueeze(-1) + x = residual + self.drop_path(x) + + # ffn + residual = x + x = self.ffn_norm(x) + x = x.permute(0, 2, 1) + x = self.ffn(x) + x = x.permute(0, 2, 1) + if self.ffn_gamma is not None: + x = x * self.ffn_gamma.unsqueeze(-1) + x = residual + self.drop_path(x) + + return x + + +class TokenizerEncoder(nn.Module): + """ + Encoder component for the VibeVoice tokenizer that converts audio to latent representations. + + Args: + config: Configuration object with model parameters + """ + def __init__(self, config): + super().__init__() + + # Extract parameters from config + self.channels = config.channels + self.dimension = config.dimension + self.n_filters = config.n_filters + self.ratios = list(reversed(config.ratios)) + self.depths = config.depths + self.n_residual_layers = getattr(config, "n_residual_layers", 1) + self.hop_length = np.prod(self.ratios) + self.causal = config.causal + + # Additional config parameters with defaults + kernel_size = getattr(config, "kernel_size", 7) + last_kernel_size = getattr(config, "last_kernel_size", 7) + norm = getattr(config, "norm", "none") + norm_params = getattr(config, "norm_params", {}) + pad_mode = getattr(config, "pad_mode", "reflect") + bias = getattr(config, "bias", True) + layernorm = getattr(config, "layernorm", "LN") + layernorm_eps = getattr(config, "layernorm_eps", 1e-6) + layernorm_elementwise_affine = getattr(config, "layernorm_elementwise_affine", True) + drop_path_rate = getattr(config, "drop_path_rate", 0.0) + mixer_layer = getattr(config, "mixer_layer", "conv") + layer_scale_init_value = getattr(config, "layer_scale_init_value", 0) + disable_last_norm = getattr(config, "disable_last_norm", False) + + # determine the norm type based on layernorm + if layernorm == 'LN': + norm_type = ConvLayerNorm + elif layernorm == 'RMSNorm': + norm_type = partial(ConvRMSNorm, elementwise_affine=layernorm_elementwise_affine) + else: + raise ValueError(f"Unsupported norm type: {layernorm}") + + # stem and intermediate downsampling conv layers + stem = nn.Sequential( + SConv1d(self.channels, self.n_filters, kernel_size, norm=norm, norm_kwargs=norm_params, causal=self.causal, pad_mode=pad_mode, bias=bias), + ) + + self.downsample_layers = nn.ModuleList() + self.downsample_layers.append(stem) + for i in range(len(self.ratios)): + in_ch = self.n_filters * (2 ** i) + out_ch = self.n_filters * (2 ** (i + 1)) + downsample_layer = nn.Sequential( + SConv1d(in_ch, out_ch, kernel_size=self.ratios[i] * 2, stride=self.ratios[i], causal=self.causal, pad_mode=pad_mode, norm=norm, bias=bias) + ) + self.downsample_layers.append(downsample_layer) + + # configure the transformer blocks + layer_type = partial( + Block1D, + mixer_layer=mixer_layer, + layernorm=layernorm, + eps=layernorm_eps, + causal=self.causal, + pad_mode=pad_mode, + norm=norm, + bias=bias, + layer_scale_init_value=layer_scale_init_value, + ) + + self.stages = nn.ModuleList() + dp_rates = [x.item() for x in torch.linspace(0, drop_path_rate, sum(self.depths))] + cur = 0 + + for i in range(len(self.depths)): + in_ch = self.n_filters * (2 ** i) + stage = nn.Sequential( + *[layer_type(dim=in_ch, drop_path=dp_rates[cur + j]) for j in range(self.depths[i])] + ) + self.stages.append(stage) + cur += self.depths[i] + + if not disable_last_norm: + self.norm = norm_type(in_ch, eps=layernorm_eps) + else: + self.norm = nn.Identity() + self.head = SConv1d(in_ch, self.dimension, kernel_size=last_kernel_size, causal=self.causal, pad_mode=pad_mode, norm=norm, bias=bias) + + def forward_features(self, x, cache=None, sample_indices=None, use_cache=False, debug=False): + for i in range(len(self.depths)): + # Apply downsampling + for layer in self.downsample_layers[i]: + if isinstance(layer, SConv1d): + x = layer(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + else: + x = layer(x) + + # Apply stage (Block1D contains Convlayer which contains SConv1d) + for block in self.stages[i]: + if hasattr(block, 'mixer') and hasattr(block.mixer, 'conv') and isinstance(block.mixer.conv, SConv1d): + # Block1D forward with cache support + residual = x + x = block.norm(x) + x = block.mixer.conv(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + if block.gamma is not None: + x = x * block.gamma.unsqueeze(-1) + x = residual + x + + # FFN part + residual = x + x = block.ffn_norm(x) + x = x.permute(0, 2, 1) + x = block.ffn(x) + x = x.permute(0, 2, 1) + if block.ffn_gamma is not None: + x = x * block.ffn_gamma.unsqueeze(-1) + x = residual + x + else: + x = block(x) + + return self.norm(x) + + def forward(self, x, cache=None, sample_indices=None, use_cache=False, debug=False): + x = self.forward_features(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + x = self.head(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + return x + + +class TokenizerDecoder(nn.Module): + """ + Decoder component for the VibeVoice tokenizer that converts latent representations back to audio. + + Args: + config: Configuration object with model parameters + """ + def __init__(self, config): + super().__init__() + + # Extract parameters from config + self.dimension = config.dimension + self.channels = config.channels + self.n_filters = config.n_filters + self.ratios = config.ratios + + # IMPORTANT CHANGE: Don't reverse depths again since they're already reversed in VibeVoiceAcousticTokenizerModel + self.depths = config.depths # Changed from list(reversed(config.depths)) + + self.n_residual_layers = getattr(config, "n_residual_layers", 1) + self.hop_length = np.prod(self.ratios) + self.causal = config.causal + + # Additional config parameters with defaults + kernel_size = getattr(config, "kernel_size", 7) + last_kernel_size = getattr(config, "last_kernel_size", 7) + norm = getattr(config, "norm", "none") + norm_params = getattr(config, "norm_params", {}) + pad_mode = getattr(config, "pad_mode", "reflect") + bias = getattr(config, "bias", True) + layernorm = getattr(config, "layernorm", "LN") + layernorm_eps = getattr(config, "layernorm_eps", 1e-6) + trim_right_ratio = getattr(config, "trim_right_ratio", 1.0) + layernorm_elementwise_affine = getattr(config, "layernorm_elementwise_affine", True) + drop_path_rate = getattr(config, "drop_path_rate", 0.0) + mixer_layer = getattr(config, "mixer_layer", "conv") + layer_scale_init_value = getattr(config, "layer_scale_init_value", 0) + disable_last_norm = getattr(config, "disable_last_norm", False) + + # determine the norm type based on layernorm + if layernorm == 'LN': + norm_type = ConvLayerNorm + elif layernorm == 'RMSNorm': + norm_type = partial(ConvRMSNorm, elementwise_affine=layernorm_elementwise_affine) + else: + raise ValueError(f"Unsupported norm type: {layernorm}") + + # stem and upsampling layers + stem = nn.Sequential( + SConv1d(self.dimension, self.n_filters * 2 ** (len(self.depths) - 1), kernel_size, norm=norm, + norm_kwargs=norm_params, causal=self.causal, pad_mode=pad_mode, bias=bias), + ) + + self.upsample_layers = nn.ModuleList() + self.upsample_layers.append(stem) + for i in range(len(self.ratios)): + in_ch = self.n_filters * (2 ** (len(self.depths) - 1 - i)) + out_ch = self.n_filters * (2 ** (len(self.depths) - 1 - i - 1)) + upsample_layer = nn.Sequential( + SConvTranspose1d(in_ch, out_ch, + kernel_size=self.ratios[i] * 2, stride=self.ratios[i], + norm=norm, norm_kwargs=norm_params, bias=bias, + causal=self.causal, trim_right_ratio=trim_right_ratio), + ) + self.upsample_layers.append(upsample_layer) + + # configure transformer blocks + layer_type = partial( + Block1D, + mixer_layer=mixer_layer, + layernorm=layernorm, + eps=layernorm_eps, + causal=self.causal, + pad_mode=pad_mode, + norm=norm, + bias=bias, + layer_scale_init_value=layer_scale_init_value, + ) + + self.stages = nn.ModuleList() + dp_rates = [x.item() for x in torch.linspace(0, drop_path_rate, sum(self.depths))] + cur = 0 + + # Create stages in the same order as the original model + for i in range(len(self.depths)): + in_ch = self.n_filters * (2 ** (len(self.depths) - 1 - i)) + stage = nn.Sequential( + *[layer_type(dim=in_ch, drop_path=dp_rates[cur + j]) for j in range(self.depths[i])] + ) + self.stages.append(stage) + cur += self.depths[i] + + if not disable_last_norm: + self.norm = norm_type(in_ch, eps=layernorm_eps) + else: + self.norm = nn.Identity() + self.head = SConv1d(in_ch, self.channels, kernel_size=last_kernel_size, causal=self.causal, pad_mode=pad_mode, norm=norm, bias=bias) + + def forward_features(self, x, cache=None, sample_indices=None, use_cache=False, debug=False): + for i in range(len(self.depths)): + # Apply upsampling + for layer in self.upsample_layers[i]: + if isinstance(layer, (SConv1d, SConvTranspose1d)): + x = layer(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + else: + x = layer(x) + + # Apply stage (Block1D contains Convlayer which contains SConv1d) + for block in self.stages[i]: + if hasattr(block, 'mixer') and hasattr(block.mixer, 'conv') and isinstance(block.mixer.conv, SConv1d): + # Block1D forward with cache support + residual = x + x = block.norm(x) + x = block.mixer.conv(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + if block.gamma is not None: + x = x * block.gamma.unsqueeze(-1) + x = residual + x + + # FFN part + residual = x + x = block.ffn_norm(x) + x = x.permute(0, 2, 1) + x = block.ffn(x) + x = x.permute(0, 2, 1) + if block.ffn_gamma is not None: + x = x * block.ffn_gamma.unsqueeze(-1) + x = residual + x + else: + x = block(x) + + return self.norm(x) + + def forward(self, x, cache=None, sample_indices=None, use_cache=False, debug=False): + x = self.forward_features(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + x = self.head(x, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + return x + + +@dataclass +class VibeVoiceTokenizerEncoderOutput: + """ + Output of VibeVoice tokenizer encoder, representing a Gaussian distribution with fixed variance. + + Args: + mean (`torch.FloatTensor`): The mean parameters of the distribution. + std (`float` or `torch.FloatTensor`): Fixed standard deviation value. + """ + mean: torch.Tensor + std: Optional[Union[float, torch.Tensor]] = None + + def sample(self, dist_type='fix'): + """ + Sample from the distribution. + + Args: + dist_type (`str`): Sampling method, either 'fix' or 'gaussian'. + + Returns: + `torch.FloatTensor`: Sampled values. + `torch.FloatTensor` (optional): Standard deviation used (only when dist_type='gaussian'). + """ + if dist_type == 'fix': + x = self.mean + self.std * torch.randn_like(self.mean) + return x, self.std + elif dist_type == 'gaussian': + batch_size = self.mean.size(0) + value = self.std / 0.8 + std = torch.randn(batch_size, device=self.mean.device, dtype=self.mean.dtype) * value + + while std.dim() < self.mean.dim(): + std = std.unsqueeze(-1) + + x = self.mean + std * torch.randn_like(self.mean) + return x, std + else: + return self.mean, self.std + + def kl(self): + """Compute KL divergence between this distribution and a standard normal.""" + target = torch.zeros_like(self.mean) + return F.mse_loss(self.mean, target, reduction='none') + + def mode(self): + """Return the distribution mode (which is the mean for Gaussian).""" + return self.mean + +class VibeVoiceAcousticTokenizerModel(PreTrainedModel): + """VibeVoice speech tokenizer model combining encoder and decoder for acoustic tokens""" + + config_class = VibeVoiceAcousticTokenizerConfig + base_model_prefix = "vibevoice_acoustic_tokenizer" + _supports_flash_attn_2 = True + _supports_sdpa = True + _no_split_modules = ["TokenizerEncoder", "TokenizerDecoder"] + + def __init__(self, config): + super().__init__(config) + + self.register_buffer('fix_std', torch.tensor(config.fix_std), persistent=False) + self.std_dist_type = getattr(config, "std_dist_type", "fix") + + # Parse encoder depths + if isinstance(config.encoder_depths, str): + encoder_depths = [int(d) for d in config.encoder_depths.split('-')] + else: + encoder_depths = config.encoder_depths + + # Parse decoder depths if provided + if config.decoder_depths is not None and isinstance(config.decoder_depths, str): + decoder_depths = [int(d) for d in config.decoder_depths.split('-')] + else: + # Default: use reversed encoder depths if decoder_depths is None + decoder_depths = list(reversed(encoder_depths)) + + # Create encoder config + encoder_config = copy.deepcopy(config) + encoder_config.dimension = config.vae_dim + encoder_config.n_filters = config.encoder_n_filters + encoder_config.ratios = config.encoder_ratios + encoder_config.depths = encoder_depths + encoder_config.norm = config.conv_norm + encoder_config.pad_mode = config.pad_mode + encoder_config.bias = config.conv_bias + encoder_config.layernorm_eps = config.layernorm_eps + encoder_config.layernorm_elementwise_affine = config.layernorm_elementwise_affine + encoder_config.mixer_layer = config.mixer_layer + encoder_config.layer_scale_init_value = config.layer_scale_init_value + encoder_config.disable_last_norm = config.disable_last_norm + + # Create decoder config + decoder_config = copy.deepcopy(config) + decoder_config.dimension = config.vae_dim + decoder_config.n_filters = config.decoder_n_filters + decoder_config.ratios = config.decoder_ratios + decoder_config.depths = decoder_depths + decoder_config.norm = config.conv_norm + decoder_config.pad_mode = config.pad_mode + decoder_config.bias = config.conv_bias + decoder_config.layernorm_eps = config.layernorm_eps + decoder_config.layernorm_elementwise_affine = config.layernorm_elementwise_affine + decoder_config.mixer_layer = config.mixer_layer + decoder_config.layer_scale_init_value = config.layer_scale_init_value + decoder_config.disable_last_norm = config.disable_last_norm + + # Initialize encoder and decoder + self.encoder = TokenizerEncoder(encoder_config) + self.decoder = TokenizerDecoder(decoder_config) + + # Initialize weights + self.apply(self._init_weights) + + def _init_weights(self, module): + """Initialize weights for the model""" + if isinstance(module, nn.Linear): + nn.init.normal_(module.weight, std=self.config.weight_init_value) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif isinstance(module, nn.LayerNorm): + nn.init.ones_(module.weight) + nn.init.zeros_(module.bias) + elif isinstance(module, nn.Conv1d): + nn.init.normal_(module.weight, std=self.config.weight_init_value) + if module.bias is not None: + nn.init.zeros_(module.bias) + + @torch.no_grad() + def encode(self, audio, cache=None, sample_indices=None, use_cache=False, debug=False): + """Convert audio to latent representations""" + latents = self.encoder(audio, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + return VibeVoiceTokenizerEncoderOutput(mean=latents.permute(0, 2, 1), std=self.fix_std) + + @torch.no_grad() + def sampling(self, encoder_output, dist_type=None): + """Sample from the encoder output distribution""" + dist_type = dist_type or self.std_dist_type + + if dist_type == 'fix': + return encoder_output.sample(dist_type='fix') + elif dist_type == 'gaussian': + return encoder_output.sample(dist_type='gaussian') + else: + raise ValueError(f"Unsupported dist_type: {dist_type}, expected 'fix' or 'gaussian'") + + @torch.no_grad() + def decode(self, latents, cache=None, sample_indices=None, use_cache=False, debug=False): + """Convert latent representations back to audio""" + if latents.shape[1] == self.config.vae_dim: + pass + else: + latents = latents.permute(0, 2, 1) + + audio = self.decoder(latents, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + return audio + + def forward(self, audio, cache=None, sample_indices=None, use_cache=False, debug=False): + """Full forward pass: encode audio to latents, then decode back to audio""" + encoder_output = self.encode(audio, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + sampled_latents, _ = self.sampling(encoder_output) + reconstructed = self.decode(sampled_latents, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + return reconstructed, sampled_latents + + +class VibeVoiceSemanticTokenizerModel(PreTrainedModel): + """VibeVoice speech tokenizer model with only encoder for semantic tokens""" + + config_class = VibeVoiceSemanticTokenizerConfig + base_model_prefix = "vibevoice_semantic_tokenizer" + _supports_flash_attn_2 = True + _supports_sdpa = True + _no_split_modules = ["TokenizerEncoder"] + + def __init__(self, config): + super().__init__(config) + + # Parse encoder depths + if isinstance(config.encoder_depths, str): + encoder_depths = [int(d) for d in config.encoder_depths.split('-')] + else: + encoder_depths = config.encoder_depths + + # Create encoder config + encoder_config = copy.deepcopy(config) + encoder_config.dimension = config.vae_dim + encoder_config.n_filters = config.encoder_n_filters + encoder_config.ratios = config.encoder_ratios + encoder_config.depths = encoder_depths + encoder_config.norm = config.conv_norm + encoder_config.pad_mode = config.pad_mode + encoder_config.bias = config.conv_bias + encoder_config.layernorm_eps = config.layernorm_eps + encoder_config.layernorm_elementwise_affine = config.layernorm_elementwise_affine + encoder_config.mixer_layer = config.mixer_layer + encoder_config.layer_scale_init_value = config.layer_scale_init_value + encoder_config.disable_last_norm = config.disable_last_norm + + # Initialize encoder and decoder + self.encoder = TokenizerEncoder(encoder_config) + + # Initialize weights + self.apply(self._init_weights) + + def _init_weights(self, module): + """Initialize weights for the model""" + if isinstance(module, nn.Linear): + nn.init.normal_(module.weight, std=self.config.weight_init_value) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif isinstance(module, nn.LayerNorm): + nn.init.ones_(module.weight) + nn.init.zeros_(module.bias) + elif isinstance(module, nn.Conv1d): + nn.init.normal_(module.weight, std=self.config.weight_init_value) + if module.bias is not None: + nn.init.zeros_(module.bias) + + @torch.no_grad() + def encode(self, audio, cache=None, sample_indices=None, use_cache=False, debug=False): + """Convert audio to latent representations""" + latents = self.encoder(audio, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + return VibeVoiceTokenizerEncoderOutput(mean=latents.permute(0, 2, 1)) + + @torch.no_grad() + def sampling(self, encoder_output, dist_type=None): + """Sample from the encoder output distribution""" + return encoder_output.sample(dist_type='none') + + def forward(self, audio, cache=None, sample_indices=None, use_cache=False, debug=False): + """Full forward pass: encode audio to latents, then decode back to audio""" + encoder_output = self.encode(audio, cache=cache, sample_indices=sample_indices, use_cache=use_cache, debug=debug) + sampled_latents, _ = self.sampling(encoder_output, dist_type='none') + return None, sampled_latents + +AutoModel.register(VibeVoiceAcousticTokenizerConfig, VibeVoiceAcousticTokenizerModel) +AutoModel.register(VibeVoiceSemanticTokenizerConfig, VibeVoiceSemanticTokenizerModel) + +__all__ = [ + "VibeVoiceTokenizerStreamingCache", + "VibeVoiceAcousticTokenizerModel", + "VibeVoiceSemanticTokenizerModel", +] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/modular/streamer.py b/VibeVoice-ComfyUI/vvembed/modular/streamer.py new file mode 100644 index 0000000000000000000000000000000000000000..6087db54fe5ff8edf9c9c0b8ad94eac1897313d1 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/modular/streamer.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import torch + +import asyncio +from queue import Queue +from typing import TYPE_CHECKING, Optional + + +from transformers.generation import BaseStreamer + + +class AudioStreamer(BaseStreamer): + """ + Audio streamer that stores audio chunks in queues for each sample in the batch. + This allows streaming audio generation for multiple samples simultaneously. + + Parameters: + batch_size (`int`): + The batch size for generation + stop_signal (`any`, *optional*): + The signal to put in the queue when generation ends. Defaults to None. + timeout (`float`, *optional*): + The timeout for the audio queue. If `None`, the queue will block indefinitely. + """ + + def __init__( + self, + batch_size: int, + stop_signal: Optional[any] = None, + timeout: Optional[float] = None, + ): + self.batch_size = batch_size + self.stop_signal = stop_signal + self.timeout = timeout + + # Create a queue for each sample in the batch + self.audio_queues = [Queue() for _ in range(batch_size)] + self.finished_flags = [False for _ in range(batch_size)] + self.sample_indices_map = {} # Maps from sample index to queue index + + def put(self, audio_chunks: torch.Tensor, sample_indices: torch.Tensor): + """ + Receives audio chunks and puts them in the appropriate queues. + + Args: + audio_chunks: Tensor of shape (num_samples, ...) containing audio chunks + sample_indices: Tensor indicating which samples these chunks belong to + """ + for i, sample_idx in enumerate(sample_indices): + idx = sample_idx.item() + if idx < self.batch_size and not self.finished_flags[idx]: + # Convert to numpy or keep as tensor based on preference + audio_chunk = audio_chunks[i].detach().cpu() + self.audio_queues[idx].put(audio_chunk, timeout=self.timeout) + + def end(self, sample_indices: Optional[torch.Tensor] = None): + """ + Signals the end of generation for specified samples or all samples. + + Args: + sample_indices: Optional tensor of sample indices to end. If None, ends all. + """ + if sample_indices is None: + # End all samples + for idx in range(self.batch_size): + if not self.finished_flags[idx]: + self.audio_queues[idx].put(self.stop_signal, timeout=self.timeout) + self.finished_flags[idx] = True + else: + # End specific samples + for sample_idx in sample_indices: + idx = sample_idx.item() if torch.is_tensor(sample_idx) else sample_idx + if idx < self.batch_size and not self.finished_flags[idx]: + self.audio_queues[idx].put(self.stop_signal, timeout=self.timeout) + self.finished_flags[idx] = True + + def __iter__(self): + """Returns an iterator over the batch of audio streams.""" + return AudioBatchIterator(self) + + def get_stream(self, sample_idx: int): + """Get the audio stream for a specific sample.""" + if sample_idx >= self.batch_size: + raise ValueError(f"Sample index {sample_idx} exceeds batch size {self.batch_size}") + return AudioSampleIterator(self, sample_idx) + + +class AudioSampleIterator: + """Iterator for a single audio stream from the batch.""" + + def __init__(self, streamer: AudioStreamer, sample_idx: int): + self.streamer = streamer + self.sample_idx = sample_idx + + def __iter__(self): + return self + + def __next__(self): + value = self.streamer.audio_queues[self.sample_idx].get(timeout=self.streamer.timeout) + if value == self.streamer.stop_signal: + raise StopIteration() + return value + + +class AudioBatchIterator: + """Iterator that yields audio chunks for all samples in the batch.""" + + def __init__(self, streamer: AudioStreamer): + self.streamer = streamer + self.active_samples = set(range(streamer.batch_size)) + + def __iter__(self): + return self + + def __next__(self): + if not self.active_samples: + raise StopIteration() + + batch_chunks = {} + samples_to_remove = set() + + # Try to get chunks from all active samples + for idx in self.active_samples: + try: + value = self.streamer.audio_queues[idx].get(block=False) + if value == self.streamer.stop_signal: + samples_to_remove.add(idx) + else: + batch_chunks[idx] = value + except: + # Queue is empty for this sample, skip it this iteration + pass + + # Remove finished samples + self.active_samples -= samples_to_remove + + if batch_chunks: + return batch_chunks + elif self.active_samples: + # If no chunks were ready but we still have active samples, + # wait a bit and try again + import time + time.sleep(0.01) + return self.__next__() + else: + raise StopIteration() + + +class AsyncAudioStreamer(AudioStreamer): + """ + Async version of AudioStreamer for use in async contexts. + """ + + def __init__( + self, + batch_size: int, + stop_signal: Optional[any] = None, + timeout: Optional[float] = None, + ): + super().__init__(batch_size, stop_signal, timeout) + # Replace regular queues with async queues + self.audio_queues = [asyncio.Queue() for _ in range(batch_size)] + self.loop = asyncio.get_running_loop() + + def put(self, audio_chunks: torch.Tensor, sample_indices: torch.Tensor): + """Put audio chunks in the appropriate async queues.""" + for i, sample_idx in enumerate(sample_indices): + idx = sample_idx.item() + if idx < self.batch_size and not self.finished_flags[idx]: + audio_chunk = audio_chunks[i].detach().cpu() + self.loop.call_soon_threadsafe( + self.audio_queues[idx].put_nowait, audio_chunk + ) + + def end(self, sample_indices: Optional[torch.Tensor] = None): + """Signal the end of generation for specified samples.""" + if sample_indices is None: + indices_to_end = range(self.batch_size) + else: + indices_to_end = [s.item() if torch.is_tensor(s) else s for s in sample_indices] + + for idx in indices_to_end: + if idx < self.batch_size and not self.finished_flags[idx]: + self.loop.call_soon_threadsafe( + self.audio_queues[idx].put_nowait, self.stop_signal + ) + self.finished_flags[idx] = True + + async def get_stream(self, sample_idx: int): + """Get async iterator for a specific sample's audio stream.""" + if sample_idx >= self.batch_size: + raise ValueError(f"Sample index {sample_idx} exceeds batch size {self.batch_size}") + + while True: + value = await self.audio_queues[sample_idx].get() + if value == self.stop_signal: + break + yield value + + def __aiter__(self): + """Returns an async iterator over all audio streams.""" + return AsyncAudioBatchIterator(self) + + +class AsyncAudioBatchIterator: + """Async iterator for batch audio streaming.""" + + def __init__(self, streamer: AsyncAudioStreamer): + self.streamer = streamer + self.active_samples = set(range(streamer.batch_size)) + + def __aiter__(self): + return self + + async def __anext__(self): + if not self.active_samples: + raise StopAsyncIteration() + + batch_chunks = {} + samples_to_remove = set() + + # Create tasks for all active samples + tasks = { + idx: asyncio.create_task(self._get_chunk(idx)) + for idx in self.active_samples + } + + # Wait for at least one chunk to be ready + done, pending = await asyncio.wait( + tasks.values(), + return_when=asyncio.FIRST_COMPLETED, + timeout=self.streamer.timeout + ) + + # Cancel pending tasks + for task in pending: + task.cancel() + + # Process completed tasks + for idx, task in tasks.items(): + if task in done: + try: + value = await task + if value == self.streamer.stop_signal: + samples_to_remove.add(idx) + else: + batch_chunks[idx] = value + except asyncio.CancelledError: + pass + + self.active_samples -= samples_to_remove + + if batch_chunks: + return batch_chunks + elif self.active_samples: + # Try again if we still have active samples + return await self.__anext__() + else: + raise StopAsyncIteration() + + async def _get_chunk(self, idx): + """Helper to get a chunk from a specific queue.""" + return await self.streamer.audio_queues[idx].get() \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/processor/__init__.py b/VibeVoice-ComfyUI/vvembed/processor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VibeVoice-ComfyUI/vvembed/processor/__pycache__/__init__.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/processor/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12dc18befeb5096905eef6ed8c28bf6aa39175c4 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/processor/__pycache__/__init__.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/processor/__pycache__/vibevoice_processor.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/processor/__pycache__/vibevoice_processor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8cd2c39564a4befb19023d963eed5050c98b0ae2 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/processor/__pycache__/vibevoice_processor.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/processor/__pycache__/vibevoice_tokenizer_processor.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/processor/__pycache__/vibevoice_tokenizer_processor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fef8c5583e2fac4bb3d83d6d786abc9644f58d9b Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/processor/__pycache__/vibevoice_tokenizer_processor.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/processor/vibevoice_processor.py b/VibeVoice-ComfyUI/vvembed/processor/vibevoice_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..1b2854c06e4e4249b3ca28be1fe7afe0e9d652e4 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/processor/vibevoice_processor.py @@ -0,0 +1,796 @@ +# Original code by Microsoft +# updated by Fabio Sarracino - Enemyx-net + +import math +import warnings +from typing import List, Optional, Union, Dict, Any, Tuple +import os +import re + +import numpy as np +import torch + +from transformers.tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy +from transformers.utils import TensorType, logging +from .vibevoice_tokenizer_processor import AudioNormalizer + +logger = logging.get_logger(__name__) + + +class VibeVoiceProcessor: + r""" + Constructs a VibeVoice processor which wraps a VibeVoice tokenizer and audio processor into a single processor. + + [`VibeVoiceProcessor`] offers all the functionalities of [`VibeVoiceTokenizer`] and [`VibeVoiceTokenizerProcessor`]. + See the [`~VibeVoiceProcessor.__call__`] and [`~VibeVoiceProcessor.decode`] for more information. + + Args: + tokenizer (`VibeVoiceTextTokenizer` or `VibeVoiceTextTokenizerFast`): + The tokenizer for text processing. + audio_processor (`VibeVoiceTokenizerProcessor`): + The audio processor for speech processing. + speech_tok_compress_ratio (`int`, *optional*, defaults to 3200): + The compression ratio for speech tokenization. + db_normalize (`bool`, *optional*, defaults to True): + Whether to apply decibel normalization to audio inputs. + """ + + def __init__(self, tokenizer=None, audio_processor=None, speech_tok_compress_ratio=3200, db_normalize=True, **kwargs): + self.tokenizer = tokenizer + self.audio_processor = audio_processor + self.speech_tok_compress_ratio = speech_tok_compress_ratio + self.db_normalize = db_normalize + self.audio_normalizer = AudioNormalizer() if db_normalize else None + self.system_prompt = " Transform the text provided by various speakers into speech output, utilizing the distinct voice of each respective speaker.\n" + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + """ + Instantiate a VibeVoiceProcessor from a pretrained VibeVoice processor. + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + - a string, the *model id* of a pretrained model + - a path to a *directory* containing processor config + + Returns: + [`VibeVoiceProcessor`]: The processor object instantiated from pretrained model. + """ + import os + import json + from .vibevoice_tokenizer_processor import VibeVoiceTokenizerProcessor + from modular.modular_vibevoice_text_tokenizer import ( + VibeVoiceTextTokenizer, + VibeVoiceTextTokenizerFast + ) + + # Load processor configuration + config_path = os.path.join(pretrained_model_name_or_path, "preprocessor_config.json") + if os.path.exists(config_path): + with open(config_path, 'r') as f: + config = json.load(f) + else: + # No preprocessor_config.json found, using defaults silently + config = { + "speech_tok_compress_ratio": 3200, + "db_normalize": True, + } + + # Extract main processor parameters + speech_tok_compress_ratio = config.get("speech_tok_compress_ratio", 3200) + db_normalize = config.get("db_normalize", True) + + # Load tokenizer - try from model path first, then fallback to Qwen + language_model_pretrained_name = config.get("language_model_pretrained_name", None) or kwargs.pop("language_model_pretrained_name", "Qwen/Qwen2.5-1.5B") + # Loading tokenizer + + # Improved tokenizer type detection + # Check if it's a Qwen tokenizer by looking at the path OR by checking the tokenizer config + is_qwen_tokenizer = False + + # First check: path contains 'qwen' + if 'qwen' in language_model_pretrained_name.lower(): + is_qwen_tokenizer = True + # Second check: if it's a local path, check the tokenizer_config.json + elif os.path.exists(language_model_pretrained_name): + tokenizer_config_path = os.path.join(language_model_pretrained_name, "tokenizer_config.json") + if os.path.exists(tokenizer_config_path): + try: + import json + with open(tokenizer_config_path, 'r') as f: + tok_config = json.load(f) + # Check if it's a Qwen tokenizer by looking at the tokenizer class or model type + if 'Qwen' in tok_config.get('tokenizer_class', '') or 'qwen' in tok_config.get('model_type', '').lower(): + is_qwen_tokenizer = True + # Detected Qwen tokenizer from config + except: + pass + + if is_qwen_tokenizer: + # Force truly local loading to avoid cache issues + # Check if this is a local directory path + if os.path.exists(language_model_pretrained_name) and os.path.isdir(language_model_pretrained_name): + # Ensure all required files exist + vocab_file = os.path.join(language_model_pretrained_name, "vocab.json") + merges_file = os.path.join(language_model_pretrained_name, "merges.txt") + tokenizer_json = os.path.join(language_model_pretrained_name, "tokenizer.json") + + # Try to load with explicit file paths first (most reliable) + if os.path.exists(vocab_file) and os.path.exists(merges_file): + try: + # Import logging if needed + import logging + logger = logging.getLogger("VibeVoice") + + # Try fast tokenizer first with tokenizer.json + if os.path.exists(tokenizer_json): + logger.info(f"Loading tokenizer with explicit files from {language_model_pretrained_name}") + logger.info(f" vocab.json: {os.path.exists(vocab_file)}") + logger.info(f" merges.txt: {os.path.exists(merges_file)}") + logger.info(f" tokenizer.json: {os.path.exists(tokenizer_json)}") + # Remove parameters that might interfere with direct file loading + clean_kwargs = {k: v for k, v in kwargs.items() + if k not in ['cache_dir', 'local_files_only', 'trust_remote_code']} + tokenizer = VibeVoiceTextTokenizerFast( + tokenizer_file=tokenizer_json, + vocab_file=vocab_file, + merges_file=merges_file, + **clean_kwargs + ) + logger.info("Tokenizer loaded successfully with explicit file paths") + else: + # Fall back to slow tokenizer if no tokenizer.json + logger.info(f"Loading slow tokenizer (no tokenizer.json found)") + # Remove parameters that might interfere with direct file loading + clean_kwargs = {k: v for k, v in kwargs.items() + if k not in ['cache_dir', 'local_files_only', 'trust_remote_code']} + tokenizer = VibeVoiceTextTokenizer( + vocab_file=vocab_file, + merges_file=merges_file, + **clean_kwargs + ) + except Exception as e: + # If direct loading fails, fall back to from_pretrained + logger.warning(f"Direct tokenizer loading failed: {e}") + logger.info("Falling back to from_pretrained method") + # But ensure we're truly using local files only + kwargs['local_files_only'] = True + kwargs.pop('cache_dir', None) # Remove cache_dir to avoid cache usage + tokenizer = VibeVoiceTextTokenizerFast.from_pretrained( + language_model_pretrained_name, + **kwargs + ) + else: + # Files missing, try from_pretrained as fallback + # BUT: first check if files exist with different check + vocab_file = os.path.join(language_model_pretrained_name, "vocab.json") + merges_file = os.path.join(language_model_pretrained_name, "merges.txt") + + if os.path.exists(vocab_file) and os.path.exists(merges_file): + # Files exist but weren't caught by first check - use direct loading + import logging + logger = logging.getLogger("VibeVoice") + logger.info(f"Retrying tokenizer load with explicit file paths") + clean_kwargs = {k: v for k, v in kwargs.items() + if k not in ['cache_dir', 'local_files_only', 'trust_remote_code']} + tokenizer = VibeVoiceTextTokenizerFast( + vocab_file=vocab_file, + merges_file=merges_file, + **clean_kwargs + ) + else: + # Truly missing files + kwargs['local_files_only'] = True + kwargs.pop('cache_dir', None) + tokenizer = VibeVoiceTextTokenizerFast.from_pretrained( + language_model_pretrained_name, + **kwargs + ) + else: + # Not a local directory, use standard from_pretrained + try: + tokenizer = VibeVoiceTextTokenizerFast.from_pretrained( + language_model_pretrained_name, + **kwargs + ) + except Exception as e: + # If from_pretrained fails, try with explicit parameters + import logging + logger = logging.getLogger("VibeVoice") + logger.warning(f"Standard from_pretrained failed: {e}") + logger.info("Trying with allow remote files...") + kwargs['local_files_only'] = False # Allow downloading + tokenizer = VibeVoiceTextTokenizerFast.from_pretrained( + language_model_pretrained_name, + **kwargs + ) + else: + raise ValueError(f"Unsupported tokenizer type for {language_model_pretrained_name}. Supported types: Qwen, Llama, Gemma.") + + # Load audio processor + if "audio_processor" in config: + # Create audio processor from config + audio_config = config["audio_processor"] + audio_processor = VibeVoiceTokenizerProcessor( + sampling_rate=audio_config.get("sampling_rate", 24000), + normalize_audio=audio_config.get("normalize_audio", True), + target_dB_FS=audio_config.get("target_dB_FS", -25), + eps=audio_config.get("eps", 1e-6), + ) + else: + # Create default audio processor + audio_processor = VibeVoiceTokenizerProcessor() + + # Create and return the processor + return cls( + tokenizer=tokenizer, + audio_processor=audio_processor, + speech_tok_compress_ratio=speech_tok_compress_ratio, + db_normalize=db_normalize, + ) + + def save_pretrained(self, save_directory: Union[str, os.PathLike], **kwargs): + """ + Save a processor to a directory, so that it can be re-loaded using the + [`~VibeVoiceProcessor.from_pretrained`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the processor will be saved. + """ + import os + import json + + os.makedirs(save_directory, exist_ok=True) + + # Save processor configuration + processor_config = { + "processor_class": "VibeVoiceProcessor", + "speech_tok_compress_ratio": self.speech_tok_compress_ratio, + "db_normalize": self.db_normalize, + "audio_processor": { + "feature_extractor_type": "VibeVoiceTokenizerProcessor", + "sampling_rate": getattr(self.audio_processor, 'sampling_rate', 24000), + "normalize_audio": getattr(self.audio_processor, 'normalize_audio', True), + "target_dB_FS": getattr(self.audio_processor, 'target_dB_FS', -25), + "eps": getattr(self.audio_processor, 'eps', 1e-6), + } + } + + config_path = os.path.join(save_directory, "preprocessor_config.json") + with open(config_path, 'w') as f: + json.dump(processor_config, f, indent=2) + + logger.info(f"Processor configuration saved in {config_path}") + + def __call__( + self, + text: Optional[Union[str, List[str], TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None, + voice_samples: Optional[Union[List[Union[str, np.ndarray]], List[List[Union[str, np.ndarray]]]]] = None, + padding: Union[bool, str, PaddingStrategy] = True, + truncation: Union[bool, str, TruncationStrategy] = False, + max_length: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_attention_mask: bool = True, + **kwargs, + ) -> BatchEncoding: + """ + Main method to process one or more podcast scripts with optional voice samples. + + Args: + text (`str`, `List[str]`): + The input text(s) to process. Can be: + - A single script string + - A list of script strings for batch processing + - A path to a .json or .txt file + - A list of paths + voice_samples (`List[Union[str, np.ndarray]]`, `List[List[Union[str, np.ndarray]]]`, *optional*): + Voice samples for each script. Can be: + - A list of samples for a single script + - A list of lists for batch processing + padding (`bool`, `str` or `PaddingStrategy`, defaults to `True`): + Whether to pad sequences to the same length + truncation (`bool`, `str` or `TruncationStrategy`, defaults to `False`): + Whether to truncate sequences + max_length (`int`, *optional*): + Maximum length of the returned sequences + return_tensors (`str` or `TensorType`, *optional*): + If set, will return tensors of a particular framework + return_attention_mask (`bool`, defaults to `True`): + Whether to return the attention mask + + Returns: + `BatchEncoding`: A BatchEncoding with the following fields: + - **input_ids** -- List of token id sequences or tensor + - **attention_mask** -- List of attention masks or tensor + - **speech_tensors** -- Padded speech inputs (if voice_samples provided) + - **speech_masks** -- Speech masks (if voice_samples provided) + - **speech_input_mask** -- Boolean masks indicating speech token positions + """ + # Handle single vs batch input + if isinstance(text, str) or (isinstance(text, list) and len(text) > 0 and not isinstance(text[0], str)): + # Single input + texts = [text] + is_batched = False + else: + # Batch input + texts = text + is_batched = True + + # Handle voice samples + if voice_samples is not None: + if not is_batched or (isinstance(voice_samples[0], (str, np.ndarray))): + # Single set of voice samples + voice_samples_list = [voice_samples] + else: + # Batch of voice samples + voice_samples_list = voice_samples + else: + voice_samples_list = [None] * len(texts) + + # Process each input + all_encodings = [] + for text_input, voice_input in zip(texts, voice_samples_list): + encoding = self._process_single(text_input, voice_input) + all_encodings.append(encoding) + + # Combine batch + batch_encoding = self._batch_encode( + all_encodings, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + return_attention_mask=return_attention_mask, + ) + + return batch_encoding + + def _process_single( + self, + text: Union[str, TextInput], + voice_samples: Optional[List[Union[str, np.ndarray]]] = None, + ) -> Dict[str, Any]: + """Process a single podcast script.""" + # Determine if text is a file path or direct script + script = None + if isinstance(text, str): + # Check if it's a file path + if text.endswith('.json') and os.path.exists(text): + script = self._convert_json_to_script(text) + elif text.endswith('.txt') and os.path.exists(text): + script = self._convert_text_to_script(text) + else: + # Assume it's the script content directly + script = text + + if script is None: + raise ValueError(f"Could not process input text: {text}") + + # Parse the script + parsed_lines = self._parse_script(script) + all_speakers = list(set(speaker_id for speaker_id, _ in parsed_lines)) + + # Create system prompt + # system_tokens = self.tokenizer.encode(self.system_prompt, add_special_tokens=False) + system_tokens = self.tokenizer.encode(self.system_prompt) + + # Process voice samples if provided + if voice_samples: + voice_tokens, voice_speech_inputs, voice_speech_masks = self._create_voice_prompt(voice_samples[:len(all_speakers)]) + else: + voice_tokens, voice_speech_inputs, voice_speech_masks = [], [], [] + + # Build full token sequence + full_tokens = system_tokens + voice_tokens + speech_input_mask = [False] * len(system_tokens) + voice_speech_masks + + # Add text input section + full_tokens += self.tokenizer.encode(' Text input:\n', add_special_tokens=False) + speech_input_mask += [False] * len(self.tokenizer.encode(' Text input:\n', add_special_tokens=False)) + + for speaker_id, speaker_text in parsed_lines: + speaker_text_tokens = self.tokenizer.encode(f" Speaker {speaker_id}:{speaker_text}\n", add_special_tokens=False) + full_tokens += speaker_text_tokens + speech_input_mask += [False] * len(speaker_text_tokens) + + # Add speech output section + full_tokens += self.tokenizer.encode(' Speech output:\n', add_special_tokens=False) + [self.tokenizer.speech_start_id] + speech_input_mask += [False] * (len(self.tokenizer.encode(' Speech output:\n', add_special_tokens=False)) + 1) + + return { + "input_ids": full_tokens, + "speech_inputs": voice_speech_inputs if voice_speech_inputs else None, + "speech_input_mask": speech_input_mask, + "parsed_script": parsed_lines, + "all_speakers": all_speakers, + } + + def _batch_encode( + self, + encodings: List[Dict[str, Any]], + padding: Union[bool, str, PaddingStrategy] = True, + truncation: Union[bool, str, TruncationStrategy] = False, + max_length: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_attention_mask: bool = True, + ) -> BatchEncoding: + """Combine multiple encodings into a batch with padding.""" + # Extract input_ids and create attention_mask + input_ids_list = [enc["input_ids"] for enc in encodings] + speech_input_masks_list = [enc["speech_input_mask"] for enc in encodings] + + # Determine padding strategy + if isinstance(padding, bool): + padding_strategy = PaddingStrategy.LONGEST if padding else PaddingStrategy.DO_NOT_PAD + elif isinstance(padding, str): + padding_strategy = PaddingStrategy(padding) + else: + padding_strategy = padding + + # Apply padding to input_ids + if padding_strategy != PaddingStrategy.DO_NOT_PAD: + if padding_strategy == PaddingStrategy.LONGEST: + max_len = max(len(ids) for ids in input_ids_list) + elif padding_strategy == PaddingStrategy.MAX_LENGTH and max_length is not None: + max_len = max_length + else: + max_len = max(len(ids) for ids in input_ids_list) + + # Pad sequences + padded_input_ids = [] + attention_masks = [] + padded_speech_input_masks = [] + + for input_ids, speech_mask in zip(input_ids_list, speech_input_masks_list): + # Truncate if needed + if truncation and len(input_ids) > max_len: + input_ids = input_ids[:max_len] + speech_mask = speech_mask[:max_len] + + # Pad + padding_length = max_len - len(input_ids) + # padded_ids = [self.tokenizer.pad_token_id] * padding_length + input_ids + padded_ids = [self.tokenizer.pad_id] * padding_length + input_ids + attention_mask = [0] * padding_length + [1] * len(input_ids) + padded_speech_mask = [False] * padding_length + speech_mask + + padded_input_ids.append(padded_ids) + attention_masks.append(attention_mask) + padded_speech_input_masks.append(padded_speech_mask) + + input_ids_list = padded_input_ids + speech_input_masks_list = padded_speech_input_masks + else: + # No padding, just create attention masks + attention_masks = [[1] * len(ids) for ids in input_ids_list] if return_attention_mask else None + + # Process speech inputs + all_speech_inputs = [] + has_speech = False + for enc in encodings: + if enc["speech_inputs"] is not None: + all_speech_inputs.extend(enc["speech_inputs"]) + has_speech = True + + # Prepare batch encoding + batch_encoding = BatchEncoding() + + # Handle tensor conversion + if return_tensors is not None: + batch_encoding["input_ids"] = torch.tensor(input_ids_list, dtype=torch.long) + if return_attention_mask and attention_masks is not None: + batch_encoding["attention_mask"] = torch.tensor(attention_masks, dtype=torch.long) + batch_encoding["speech_input_mask"] = torch.tensor(speech_input_masks_list, dtype=torch.bool) + else: + batch_encoding["input_ids"] = input_ids_list + if return_attention_mask and attention_masks is not None: + batch_encoding["attention_mask"] = attention_masks + batch_encoding["speech_input_mask"] = speech_input_masks_list + + # Process speech tensors if present + if has_speech: + speech_dict = self.prepare_speech_inputs( + all_speech_inputs, + return_tensors=return_tensors, + ) + batch_encoding["speech_tensors"] = speech_dict["padded_speeches"] + batch_encoding["speech_masks"] = speech_dict["speech_masks"] + else: + batch_encoding["speech_tensors"] = None + batch_encoding["speech_masks"] = None + + # Add metadata + batch_encoding["parsed_scripts"] = [enc["parsed_script"] for enc in encodings] + batch_encoding["all_speakers_list"] = [enc["all_speakers"] for enc in encodings] + + return batch_encoding + + def _create_voice_prompt( + self, + speaker_samples: List[Union[str, np.ndarray]] + ) -> Tuple[List[int], List[np.ndarray], List[bool]]: + """ + Create voice prompt tokens and process audio samples. + + Returns: + tuple: (voice_tokens, voice_speech_inputs, voice_speech_masks) + """ + vae_token_id = self.tokenizer.speech_diffusion_id + + voice_full_tokens = self.tokenizer.encode(' Voice input:\n', add_special_tokens=False) + voice_speech_inputs = [] + voice_speech_masks = [False] * len(voice_full_tokens) + + for speaker_id, speaker_audio in enumerate(speaker_samples): + prefix_tokens = self.tokenizer.encode(f" Speaker {speaker_id}:", add_special_tokens=False) + + # Process audio + if isinstance(speaker_audio, str): + # Load audio from file + wav = self.audio_processor._load_audio_from_path(speaker_audio) + else: + wav = np.array(speaker_audio, dtype=np.float32) + + # Apply normalization if needed + if self.db_normalize and self.audio_normalizer: + wav = self.audio_normalizer(wav) + + # Calculate token length based on compression ratio + # if speaker_audio.endswith('.pt') or speaker_audio.endswith('.npy'): + # vae_tok_len = wav.shape[0] + # else: + vae_tok_len = math.ceil(wav.shape[0] / self.speech_tok_compress_ratio) + + # Build tokens and masks + speaker_tokens = (prefix_tokens + + [self.tokenizer.speech_start_id] + + [vae_token_id] * vae_tok_len + + [self.tokenizer.speech_end_id] + + self.tokenizer.encode('\n', add_special_tokens=False)) + + vae_input_mask = ([False] * len(prefix_tokens) + + [False] + + [True] * vae_tok_len + + [False] + + [False]) + + voice_full_tokens.extend(speaker_tokens) + voice_speech_masks.extend(vae_input_mask) + voice_speech_inputs.append(wav) + + return voice_full_tokens, voice_speech_inputs, voice_speech_masks + + def prepare_speech_inputs( + self, + speech_inputs: List[np.ndarray], + return_tensors: Optional[Union[str, TensorType]] = None, + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, + ) -> Dict[str, Any]: + """ + Prepare speech inputs for model consumption. + + Args: + speech_inputs: List of speech arrays + return_tensors: Output tensor type + device: Device to place tensors on + dtype: Data type for tensors + + Returns: + Dictionary with padded_speeches and speech_masks + """ + if not speech_inputs: + return {"padded_speeches": None, "speech_masks": None} + + # Calculate sequence lengths + vae_tok_seqlens = [math.ceil(s.shape[0] / self.speech_tok_compress_ratio) for s in speech_inputs] + # vae_tok_seqlens = [math.ceil(s.shape[0] / self.speech_tok_compress_ratio) if s.ndim == 1 else s.shape[0] for s in speech_inputs] + max_speech_length = max(s.shape[0] for s in speech_inputs) + + # Pad speeches + if speech_inputs[0].ndim == 1: + padded_speeches = np.full((len(speech_inputs), max_speech_length), fill_value=0, dtype=np.float32) + else: + padded_speeches = np.full((len(speech_inputs), max_speech_length, speech_inputs[0].shape[-1]), fill_value=0, dtype=np.float32) + speech_masks = np.zeros((len(speech_inputs), max(vae_tok_seqlens)), dtype=np.bool_) + + for i, (speech, vae_tok_length) in enumerate(zip(speech_inputs, vae_tok_seqlens)): + padded_speeches[i, :len(speech)] = speech + speech_masks[i, :vae_tok_length] = True + + result = { + "padded_speeches": padded_speeches, + "speech_masks": speech_masks, + } + + # Convert to tensors if requested + if return_tensors == "pt": + result["padded_speeches"] = torch.tensor(padded_speeches, device=device, dtype=dtype or torch.float32) + result["speech_masks"] = torch.tensor(speech_masks, device=device, dtype=torch.bool) + + return result + + def _convert_json_to_script(self, json_file: str) -> str: + """ + Convert JSON format to script format. + Expected JSON format: + [ + {"speaker": "1", "text": "Hello everyone..."}, + {"speaker": "2", "text": "Great to be here..."} + ] + """ + import json + + with open(json_file, 'r', encoding='utf-8') as f: + data = json.load(f) + + if not isinstance(data, list): + raise ValueError("JSON file must contain a list of speaker entries") + + script_lines = [] + for item in data: + if not isinstance(item, dict): + logger.warning(f"Skipping non-dict entry: {item}") + continue + + speaker = item.get('speaker') + text = item.get('text') + + if speaker is None or text is None: + logger.warning(f"Skipping entry missing speaker or text: {item}") + continue + + # Ensure speaker ID is valid + try: + speaker_id = int(speaker) + except (ValueError, TypeError): + logger.warning(f"Invalid speaker ID: {speaker}, skipping entry") + continue + + # Clean up text + text = text.strip() + if text: + script_lines.append(f"Speaker {speaker_id}: {text}") + + if not script_lines: + raise ValueError("No valid entries found in JSON file") + + return "\n".join(script_lines) + + def _convert_text_to_script(self, text_file: str) -> str: + """ + Convert text file to script format. + Handles multiple formats: + 1. Already formatted as "Speaker X: text" + 2. Plain text (assigns to Speaker 1) + + Handles edge cases like multiple colons in a line. + """ + with open(text_file, 'r', encoding='utf-8') as f: + lines = f.readlines() + + script_lines = [] + current_speaker = 1 + + for line in lines: + line = line.strip() + if not line: + continue + + # Try to parse as "Speaker X: text" format + # Use regex to be more robust + speaker_match = re.match(r'^Speaker\s+(\d+)\s*:\s*(.*)$', line, re.IGNORECASE) + + if speaker_match: + speaker_id = int(speaker_match.group(1)) + text = speaker_match.group(2).strip() + if text: + script_lines.append(f"Speaker {speaker_id}: {text}") + else: + # Treat as plain text - assign to current speaker + script_lines.append(f"Speaker {current_speaker}: {line}") + + if not script_lines: + raise ValueError("No valid content found in text file") + + return "\n".join(script_lines) + + def _parse_script(self, script: str) -> List[Tuple[int, str]]: + """Parse script into list of (speaker_id, text) tuples.""" + lines = script.strip().split("\n") + parsed_lines = [] + speaker_ids = [] + + # First pass: parse all lines and collect speaker IDs + for line in lines: + if not line.strip(): + continue + + # Use regex to handle edge cases like multiple colons + match = re.match(r'^Speaker\s+(\d+)\s*:\s*(.*)$', line.strip(), re.IGNORECASE) + + if match: + speaker_id = int(match.group(1)) + text = ' ' + match.group(2).strip() + parsed_lines.append((speaker_id, text)) + speaker_ids.append(speaker_id) + else: + logger.warning(f"Could not parse line: '{line}'") + + if not parsed_lines: + raise ValueError("No valid speaker lines found in script") + + # Check if we need to normalize speaker IDs (only if all are > 0) + min_speaker_id = min(speaker_ids) + if min_speaker_id > 0: + # Normalize to start from 0 + normalized_lines = [] + for speaker_id, text in parsed_lines: + normalized_lines.append((speaker_id - 1, text)) + return normalized_lines + else: + # Keep original IDs + return parsed_lines + + def _merge_inputs(self, text_inputs: BatchEncoding, audio_inputs: Dict) -> BatchEncoding: + """Merge text and audio inputs into a single BatchEncoding.""" + # Start with text inputs + merged = BatchEncoding(text_inputs) + + # Add audio-specific fields + if "audio" in audio_inputs: + merged["speech_inputs"] = audio_inputs["audio"] + if "streaming" in audio_inputs: + merged["streaming"] = audio_inputs["streaming"] + + return merged + + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to VibeVoiceTextTokenizer's [`~PreTrainedTokenizer.batch_decode`]. + Please refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to VibeVoiceTextTokenizer's [`~PreTrainedTokenizer.decode`]. + Please refer to the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + """ + Return the list of inputs accepted by the model. + """ + tokenizer_input_names = self.tokenizer.model_input_names + audio_processor_input_names = self.audio_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + audio_processor_input_names + ["speech_inputs", "speech_input_mask"])) + + def save_audio(self, + audio: Union[torch.Tensor, np.ndarray, List[Union[torch.Tensor, np.ndarray]]], + output_path: str = "output.wav", + sampling_rate: Optional[int] = None, + normalize: bool = False, + batch_prefix: str = "audio_", + ) -> str: + """ + Save audio data to a file. + Args: + audio (Union[torch.Tensor, np.ndarray, List[Union[torch.Tensor, np.ndarray]]]): + The audio data to save. Can be a single tensor/array or a list of them. + output_path (str, optional): Path to save the audio file. Defaults to "output.wav". + sampling_rate (int, optional): Sampling rate for the audio. If None, uses the processor's default. + normalize (bool, optional): Whether to normalize the audio before saving. Defaults to False. + batch_prefix (str, optional): Prefix for batch audio files. Defaults to "audio_". + Returns: + str: The path to the saved audio file. + """ + return self.audio_processor.save_audio(audio, output_path=output_path, sampling_rate=sampling_rate, normalize=normalize, batch_prefix=batch_prefix) + +__all__ = [ + "VibeVoiceProcessor", +] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/processor/vibevoice_tokenizer_processor.py b/VibeVoice-ComfyUI/vvembed/processor/vibevoice_tokenizer_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..54c40a388719b086dec65c4ebd21fe5e1f50a493 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/processor/vibevoice_tokenizer_processor.py @@ -0,0 +1,483 @@ +""" +Processor class for VibeVoice models. +""" + +import os +import json +import warnings +from typing import List, Optional, Union, Dict, Any + +import numpy as np +import torch + +from transformers.feature_extraction_utils import FeatureExtractionMixin +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +class AudioNormalizer: + """ + Audio normalization class for VibeVoice tokenizer. + + This class provides audio normalization to ensure consistent input levels + for the VibeVoice tokenizer while maintaining audio quality. + """ + + def __init__(self, target_dB_FS: float = -25, eps: float = 1e-6): + """ + Initialize the audio normalizer. + + Args: + target_dB_FS (float): Target dB FS level for the audio. Default: -25 + eps (float): Small value to avoid division by zero. Default: 1e-6 + """ + self.target_dB_FS = target_dB_FS + self.eps = eps + + def tailor_dB_FS(self, audio: np.ndarray) -> tuple: + """ + Adjust the audio to the target dB FS level. + + Args: + audio (np.ndarray): Input audio signal + + Returns: + tuple: (normalized_audio, rms, scalar) + """ + rms = np.sqrt(np.mean(audio**2)) + scalar = 10 ** (self.target_dB_FS / 20) / (rms + self.eps) + normalized_audio = audio * scalar + return normalized_audio, rms, scalar + + def avoid_clipping(self, audio: np.ndarray, scalar: Optional[float] = None) -> tuple: + """ + Avoid clipping by scaling down if necessary. + + Args: + audio (np.ndarray): Input audio signal + scalar (float, optional): Explicit scaling factor + + Returns: + tuple: (normalized_audio, scalar) + """ + if scalar is None: + max_val = np.max(np.abs(audio)) + if max_val > 1.0: + scalar = max_val + self.eps + else: + scalar = 1.0 + + return audio / scalar, scalar + + def __call__(self, audio: np.ndarray) -> np.ndarray: + """ + Normalize the audio by adjusting to target dB FS and avoiding clipping. + + Args: + audio (np.ndarray): Input audio signal + + Returns: + np.ndarray: Normalized audio signal + """ + # First adjust to target dB FS + audio, _, _ = self.tailor_dB_FS(audio) + # Then avoid clipping + audio, _ = self.avoid_clipping(audio) + return audio + + +# Change from ProcessorMixin to FeatureExtractionMixin which is designed for single components +class VibeVoiceTokenizerProcessor(FeatureExtractionMixin): + """ + Processor for VibeVoice acoustic tokenizer models. + + This processor handles audio preprocessing for VibeVoice models, including: + - Audio format conversion (stereo to mono) + - Optional audio normalization + - Streaming support for infinite-length audio + + Args: + sampling_rate (int, optional): Expected sampling rate. Defaults to 24000. + normalize_audio (bool, optional): Whether to normalize audio. Defaults to True. + target_dB_FS (float, optional): Target dB FS for normalization. Defaults to -25. + eps (float, optional): Small value for numerical stability. Defaults to 1e-6. + """ + model_input_names = ["input_features"] + + def __init__( + self, + sampling_rate: int = 24000, + normalize_audio: bool = True, + target_dB_FS: float = -25, + eps: float = 1e-6, + **kwargs, + ): + super().__init__(**kwargs) + + self.sampling_rate = sampling_rate + self.normalize_audio = normalize_audio + + # Initialize audio normalizer if needed + if self.normalize_audio: + self.normalizer = AudioNormalizer(target_dB_FS=target_dB_FS, eps=eps) + else: + self.normalizer = None + + # Save config + self.feature_extractor_dict = { + "sampling_rate": sampling_rate, + "normalize_audio": normalize_audio, + "target_dB_FS": target_dB_FS, + "eps": eps, + } + + def _ensure_mono(self, audio: np.ndarray) -> np.ndarray: + """ + Convert stereo audio to mono if needed. + + Args: + audio (np.ndarray): Input audio array + + Returns: + np.ndarray: Mono audio array + """ + if len(audio.shape) == 1: + return audio + elif len(audio.shape) == 2: + if audio.shape[0] == 2: # (2, time) + return np.mean(audio, axis=0) + elif audio.shape[1] == 2: # (time, 2) + return np.mean(audio, axis=1) + else: + # If one dimension is 1, squeeze it + if audio.shape[0] == 1: + return audio.squeeze(0) + elif audio.shape[1] == 1: + return audio.squeeze(1) + else: + raise ValueError(f"Unexpected audio shape: {audio.shape}") + else: + raise ValueError(f"Audio should be 1D or 2D, got shape: {audio.shape}") + + def _process_single_audio(self, audio: Union[np.ndarray, List[float]]) -> np.ndarray: + """ + Process a single audio array. + + Args: + audio: Single audio input + + Returns: + np.ndarray: Processed audio + """ + # Convert to numpy array + if not isinstance(audio, np.ndarray): + audio = np.array(audio, dtype=np.float32) + else: + audio = audio.astype(np.float32) + + # Ensure mono + audio = self._ensure_mono(audio) + + # Normalize if requested + if self.normalize_audio and self.normalizer is not None: + audio = self.normalizer(audio) + + return audio + + def __call__( + self, + audio: Union[str, np.ndarray, List[float], List[np.ndarray], List[List[float]], List[str]] = None, + sampling_rate: Optional[int] = None, + return_tensors: Optional[str] = None, + **kwargs, + ): + """ + Process audio for VibeVoice models. + + Args: + audio: Audio input(s) to process. Can be: + - str: Path to audio file + - np.ndarray: Audio array + - List[float]: Audio as list of floats + - List[np.ndarray]: Batch of audio arrays + - List[str]: Batch of audio file paths + sampling_rate (int, optional): Sampling rate of the input audio + return_tensors (str, optional): Return format ('pt' for PyTorch, 'np' for NumPy) + + Returns: + dict: Processed audio inputs with keys: + - input_features: Audio tensor(s) ready for the model + """ + if audio is None: + raise ValueError("Audio input is required") + + # Validate sampling rate + if sampling_rate is not None and sampling_rate != self.sampling_rate: + logger.warning( + f"Input sampling rate ({sampling_rate}) differs from expected " + f"sampling rate ({self.sampling_rate}). Please resample your audio." + ) + + # Handle different input types + if isinstance(audio, str): + # Single audio file path + audio = self._load_audio_from_path(audio) + is_batched = False + elif isinstance(audio, list): + if len(audio) == 0: + raise ValueError("Empty audio list provided") + + # Check if it's a list of file paths + if all(isinstance(item, str) for item in audio): + # Batch of audio file paths + audio = [self._load_audio_from_path(path) for path in audio] + is_batched = True + else: + # Check if it's batched audio arrays + is_batched = isinstance(audio[0], (np.ndarray, list)) + else: + # Single audio array or list + is_batched = False + + # Process audio + if is_batched: + processed_audio = [self._process_single_audio(a) for a in audio] + else: + processed_audio = [self._process_single_audio(audio)] + + # Convert to tensors if requested + if return_tensors == "pt": + if len(processed_audio) == 1: + # Create a proper batch dimension (B, T) + input_features = torch.from_numpy(processed_audio[0]).unsqueeze(0).unsqueeze(1) + else: + # For batched input with different lengths, create a batch properly + input_features = torch.stack([torch.from_numpy(a) for a in processed_audio]).unsqueeze(1) + elif return_tensors == "np": + if len(processed_audio) == 1: + input_features = processed_audio[0][np.newaxis, np.newaxis, :] + else: + input_features = np.stack(processed_audio)[:, np.newaxis, :] + else: + input_features = processed_audio[0] if len(processed_audio) == 1 else processed_audio + + outputs = { + "audio": input_features, # Use "audio" instead of "input_features" + } + + return outputs + + def _load_audio_from_path(self, audio_path: str) -> np.ndarray: + """ + Load audio from file path. + + Args: + audio_path (str): Path to audio file + + Returns: + np.ndarray: Loaded audio array + """ + # Get file extension to determine loading method + file_ext = os.path.splitext(audio_path)[1].lower() + + if file_ext in ['.wav', '.mp3', '.flac', '.m4a', '.ogg']: + # Audio file - use librosa + import librosa + audio_array, sr = librosa.load( + audio_path, + sr=self.sampling_rate, + mono=True + ) + return audio_array + elif file_ext == '.pt': + # PyTorch tensor file + audio_tensor = torch.load(audio_path, map_location='cpu').squeeze() + if isinstance(audio_tensor, torch.Tensor): + audio_array = audio_tensor.numpy() + else: + audio_array = np.array(audio_tensor) + return audio_array.astype(np.float32) + elif file_ext == '.npy': + # NumPy file + audio_array = np.load(audio_path) + return audio_array.astype(np.float32) + else: + raise ValueError( + f"Unsupported file format: {file_ext}. " + f"Supported formats: .wav, .mp3, .flac, .m4a, .ogg, .pt, .npy, .npz" + ) + + def preprocess_audio( + self, + audio_path_or_array: Union[str, np.ndarray], + normalize: Optional[bool] = None, + ) -> np.ndarray: + """ + Convenience method to preprocess audio from file path or array. + This method is kept for backward compatibility but __call__ is recommended. + + Args: + audio_path_or_array: Path to audio file or numpy array + normalize: Whether to normalize (overrides default setting) + + Returns: + np.ndarray: Preprocessed audio array + """ + if isinstance(audio_path_or_array, str): + audio_array = self._load_audio_from_path(audio_path_or_array) + else: + audio_array = np.array(audio_path_or_array, dtype=np.float32) + + # Override normalization setting if specified + original_normalize = self.normalize_audio + if normalize is not None: + self.normalize_audio = normalize + + try: + processed = self._process_single_audio(audio_array) + finally: + # Restore original setting + self.normalize_audio = original_normalize + + return processed + + # Override to_dict method for configuration saving + def to_dict(self) -> Dict[str, Any]: + """ + Convert the object to a dict containing all attributes needed for serialization. + """ + return self.feature_extractor_dict + + def save_audio( + self, + audio: Union[torch.Tensor, np.ndarray, List[Union[torch.Tensor, np.ndarray]]], + output_path: str = "output.wav", + sampling_rate: Optional[int] = None, + normalize: bool = False, + batch_prefix: str = "audio_", + ): + """ + Save audio data to WAV file(s). + + Args: + audio: Audio data to save. Can be: + - torch.Tensor: PyTorch tensor with shape (B, C, T) or (B, T) or (T) + - np.ndarray: NumPy array with shape (B, C, T) or (B, T) or (T) + - List of tensors or arrays + output_path: Path where to save the audio. If saving multiple files, + this is treated as a directory and individual files will be saved inside. + sampling_rate: Sampling rate for the saved audio. Defaults to the processor's rate. + normalize: Whether to normalize audio before saving. + batch_prefix: Prefix for batch files when saving multiple audios. + + Returns: + List[str]: Paths to the saved audio files. + """ + if sampling_rate is None: + sampling_rate = self.sampling_rate + + try: + import soundfile as sf + except ImportError: + raise ImportError( + "soundfile is required to save audio files. " + "Install it with: pip install soundfile" + ) + + # Ensure audio is in the right format + if isinstance(audio, torch.Tensor): + # Convert PyTorch tensor to numpy + audio_np = audio.float().detach().cpu().numpy() + elif isinstance(audio, np.ndarray): + audio_np = audio + elif isinstance(audio, list): + # Handle list of tensors or arrays + if all(isinstance(a, torch.Tensor) for a in audio): + audio_np = [a.float().detach().cpu().numpy() for a in audio] + else: + audio_np = audio + else: + raise ValueError(f"Unsupported audio type: {type(audio)}") + + saved_paths = [] + + # Handle based on shape or type + if isinstance(audio_np, list): + # Multiple separate audios to save + output_dir = output_path + + # Ensure output directory exists + os.makedirs(output_dir, exist_ok=True) + + # Save each audio + for i, audio_item in enumerate(audio_np): + audio_item = self._prepare_audio_for_save(audio_item, normalize) + file_path = os.path.join(output_dir, f"{batch_prefix}{i}.wav") + sf.write(file_path, audio_item, sampling_rate) + saved_paths.append(file_path) + + else: + # Handle different dimensions + if len(audio_np.shape) >= 3: # (B, C, T) or similar + # Get batch size + batch_size = audio_np.shape[0] + + if batch_size > 1: + # Multiple audios in a batch + output_dir = output_path + + # Ensure output directory exists + os.makedirs(output_dir, exist_ok=True) + + # Save each audio in the batch + for i in range(batch_size): + # Extract single audio and remove channel dim if present + single_audio = audio_np[i] + if len(single_audio.shape) > 1: + if single_audio.shape[0] == 1: # (1, T) + single_audio = single_audio.squeeze(0) + + single_audio = self._prepare_audio_for_save(single_audio, normalize) + file_path = os.path.join(output_dir, f"{batch_prefix}{i}.wav") + sf.write(file_path, single_audio, sampling_rate) + saved_paths.append(file_path) + else: + # Single audio with batch and channel dims + audio_item = audio_np.squeeze() # Remove batch and channel dimensions + audio_item = self._prepare_audio_for_save(audio_item, normalize) + sf.write(output_path, audio_item, sampling_rate) + saved_paths.append(output_path) + else: + # Single audio without batch dimension + audio_item = self._prepare_audio_for_save(audio_np, normalize) + sf.write(output_path, audio_item, sampling_rate) + saved_paths.append(output_path) + + return saved_paths + + def _prepare_audio_for_save(self, audio: np.ndarray, normalize: bool) -> np.ndarray: + """ + Prepare audio for saving by ensuring it's the right shape and optionally normalizing. + + Args: + audio: Audio data as numpy array + normalize: Whether to normalize audio + + Returns: + np.ndarray: Processed audio ready for saving + """ + # Ensure right dimensionality + if len(audio.shape) > 1 and audio.shape[0] == 1: # (1, T) + audio = audio.squeeze(0) + + # Normalize if requested + if normalize: + max_val = np.abs(audio).max() + if max_val > 0: + audio = audio / max_val + + return audio + + +__all__ = ["VibeVoiceTokenizerProcessor", "AudioNormalizer"] \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/schedule/__init__.py b/VibeVoice-ComfyUI/vvembed/schedule/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VibeVoice-ComfyUI/vvembed/schedule/__pycache__/__init__.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/schedule/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cec2e68b660f5861e4ccf851603d2ac4b6fce24 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/schedule/__pycache__/__init__.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/schedule/__pycache__/dpm_solver.cpython-312.pyc b/VibeVoice-ComfyUI/vvembed/schedule/__pycache__/dpm_solver.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..06f891ab5a9b16486be247cac97154e6833d4eb0 Binary files /dev/null and b/VibeVoice-ComfyUI/vvembed/schedule/__pycache__/dpm_solver.cpython-312.pyc differ diff --git a/VibeVoice-ComfyUI/vvembed/schedule/dpm_solver.py b/VibeVoice-ComfyUI/vvembed/schedule/dpm_solver.py new file mode 100644 index 0000000000000000000000000000000000000000..ed7f1f1bb19f99fb697b119da5ea7f1af2fd0684 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/schedule/dpm_solver.py @@ -0,0 +1,1065 @@ +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import deprecate +from diffusers.utils.torch_utils import randn_tensor +from diffusers.schedulers.scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + # return math.cos(t * math.pi / 2 * 0.95) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + elif alpha_transform_type == "cauchy": + # µ + γ tan (π (0.5 - x)) γ = 1, µ = 3 + # alpha^2 = 1-1/(exp(λ)+1) + def alpha_bar_fn(t, gamma=1, mu=3): + snr = mu + gamma * math.tan(math.pi * (0.5 - t) * 0.9) + return 1 - 1 / (math.exp(snr) + 1.1) + + elif alpha_transform_type == "laplace": + # µ − bsgn(0.5 − t) log(1 − 2|t − 0.5|) µ = 0, b = 1 + def alpha_bar_fn(t, mu=0, b=1): + snr = mu - b * math.copysign(1, 0.5 - t) * math.log(1 - 2 * abs(t - 0.5) * 0.98) + return 1 - 1 / (math.exp(snr) + 1.02) + + else: + raise ValueError(f"Unsupported alpha_transform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.Tensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.Tensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + +class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): + """ + `DPMSolverMultistepScheduler` is a fast dedicated high-order solver for diffusion ODEs. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, defaults to 2): + The DPMSolver order which can be `1` or `2` or `3`. It is recommended to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++"`. + algorithm_type (`str`, defaults to `dpmsolver++`): + Algorithm type for the solver; can be `dpmsolver`, `dpmsolver++`, `sde-dpmsolver` or `sde-dpmsolver++`. The + `dpmsolver` type implements the algorithms in the [DPMSolver](https://huggingface.co/papers/2206.00927) + paper, and the `dpmsolver++` type implements the algorithms in the + [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or + `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion. + solver_type (`str`, defaults to `midpoint`): + Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the + sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + euler_at_final (`bool`, defaults to `False`): + Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail + richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference + steps, but sometimes may result in blurring. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + use_lu_lambdas (`bool`, *optional*, defaults to `False`): + Whether to use the uniform-logSNR for step sizes proposed by Lu's DPM-Solver in the noise schedule during + the sampling process. If `True`, the sigmas and time steps are determined according to a sequence of + `lambda(t)`. + final_sigmas_type (`str`, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final + sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + lambda_min_clipped (`float`, defaults to `-inf`): + Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the + cosine (`squaredcos_cap_v2`) noise schedule. + variance_type (`str`, *optional*): + Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output + contains the predicted Gaussian variance. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = True, + euler_at_final: bool = False, + use_karras_sigmas: Optional[bool] = False, + use_lu_lambdas: Optional[bool] = False, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + lambda_min_clipped: float = -float("inf"), + variance_type: Optional[str] = None, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + rescale_betas_zero_snr: bool = False, + ): + if algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + deprecation_message = f"algorithm_type {algorithm_type} is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead" + deprecate("algorithm_types dpmsolver and sde-dpmsolver", "1.0.0", deprecation_message) + + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2" or beta_schedule == "cosine": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps, alpha_transform_type="cosine") + elif beta_schedule == "cauchy": + self.betas = betas_for_alpha_bar(num_train_timesteps, alpha_transform_type="cauchy") + elif beta_schedule == "laplace": + self.betas = betas_for_alpha_bar(num_train_timesteps, alpha_transform_type="laplace") + else: + raise NotImplementedError(f"{beta_schedule} is not implemented for {self.__class__}") + + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + if rescale_betas_zero_snr: + # Close to 0 without being 0 so first sigma is not inf + # FP16 smallest positive subnormal works well here + self.alphas_cumprod[-1] = 2**-24 + + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # settings for DPM-Solver + if algorithm_type not in ["dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++"]: + if algorithm_type == "deis": + self.register_to_config(algorithm_type="dpmsolver++") + else: + raise NotImplementedError(f"{algorithm_type} is not implemented for {self.__class__}") + + if solver_type not in ["midpoint", "heun"]: + if solver_type in ["logrho", "bh1", "bh2"]: + self.register_to_config(solver_type="midpoint") + else: + raise NotImplementedError(f"{solver_type} is not implemented for {self.__class__}") + + if algorithm_type not in ["dpmsolver++", "sde-dpmsolver++"] and final_sigmas_type == "zero": + raise ValueError( + f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please choose `sigma_min` instead." + ) + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.lower_order_nums = 0 + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps( + self, + num_inference_steps: int = None, + device: Union[str, torch.device] = None, + timesteps: Optional[List[int]] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary timesteps schedule. If `None`, timesteps will be generated + based on the `timestep_spacing` attribute. If `timesteps` is passed, `num_inference_steps` and `sigmas` + must be `None`, and `timestep_spacing` attribute will be ignored. + """ + if num_inference_steps is None and timesteps is None: + raise ValueError("Must pass exactly one of `num_inference_steps` or `timesteps`.") + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.") + if timesteps is not None and self.config.use_karras_sigmas: + raise ValueError("Cannot use `timesteps` with `config.use_karras_sigmas = True`") + if timesteps is not None and self.config.use_lu_lambdas: + raise ValueError("Cannot use `timesteps` with `config.use_lu_lambdas = True`") + + if timesteps is not None: + timesteps = np.array(timesteps).astype(np.int64) + else: + # Clipping the minimum of all lambda(t) for numerical stability. + # This is critical for cosine (squaredcos_cap_v2) noise schedule. + clipped_idx = torch.searchsorted(torch.flip(self.lambda_t, [0]), self.config.lambda_min_clipped) + last_timestep = ((self.config.num_train_timesteps - clipped_idx).numpy()).item() + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, last_timestep - 1, num_inference_steps + 1) + .round()[::-1][:-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = last_timestep // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = ( + (np.arange(0, num_inference_steps + 1) * step_ratio).round()[::-1][:-1].copy().astype(np.int64) + ) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.arange(last_timestep, 0, -step_ratio).round().copy().astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + + if self.config.use_karras_sigmas: + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + elif self.config.use_lu_lambdas: + lambdas = np.flip(log_sigmas.copy()) + lambdas = self._convert_to_lu(in_lambdas=lambdas, num_inference_steps=num_inference_steps) + sigmas = np.exp(lambdas) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.Tensor, num_inference_steps) -> torch.Tensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def _convert_to_lu(self, in_lambdas: torch.Tensor, num_inference_steps) -> torch.Tensor: + """Constructs the noise schedule of Lu et al. (2022).""" + + lambda_min: float = in_lambdas[-1].item() + lambda_max: float = in_lambdas[0].item() + + rho = 1.0 # 1.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = lambda_min ** (1 / rho) + max_inv_rho = lambda_max ** (1 / rho) + lambdas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return lambdas + + def convert_model_output( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + """ + Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is + designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an + integral of the data prediction model. + + + + The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise + prediction and data prediction models. + + + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.Tensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + # DPM-Solver++ needs to solve an integral of the data prediction model. + if self.config.algorithm_type in ["dpmsolver++", "sde-dpmsolver++"]: + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + model_output = model_output[:, :3] + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + + # DPM-Solver needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + epsilon = model_output[:, :3] + else: + epsilon = model_output + elif self.config.prediction_type == "sample": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = (sample - alpha_t * model_output) / sigma_t + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = alpha_t * model_output + sigma_t * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * epsilon) / alpha_t + x0_pred = self._threshold_sample(x0_pred) + epsilon = (sample - alpha_t * x0_pred) / sigma_t + + return epsilon + + def dpm_solver_first_order_update( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + noise: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the first-order DPMSolver (equivalent to DDIM). + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / sigma_s) * sample - (alpha_t * (torch.exp(-h) - 1.0)) * model_output + elif self.config.algorithm_type == "dpmsolver": + x_t = (alpha_t / alpha_s) * sample - (sigma_t * (torch.exp(h) - 1.0)) * model_output + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + x_t = ( + (sigma_t / sigma_s * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + x_t = ( + (alpha_t / alpha_s) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * model_output + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + return x_t + + def multistep_dpm_solver_second_order_update( + self, + model_output_list: List[torch.Tensor], + *args, + sample: torch.Tensor = None, + noise: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the second-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.Tensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + - 0.5 * (alpha_t * (torch.exp(-h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 0.5 * (sigma_t * (torch.exp(h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + ) + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + 0.5 * (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + (alpha_t * ((1.0 - torch.exp(-2.0 * h)) / (-2.0 * h) + 1.0)) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s0) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * (torch.exp(h) - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s0) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 2.0 * (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + return x_t + + def multistep_dpm_solver_third_order_update( + self, + model_output_list: List[torch.Tensor], + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the third-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.Tensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.Tensor`): + A current instance of a sample created by diffusion process. + + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing`sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1, sigma_s2 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + self.sigmas[self.step_index - 2], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + lambda_s2 = torch.log(alpha_s2) - torch.log(sigma_s2) + + m0, m1, m2 = model_output_list[-1], model_output_list[-2], model_output_list[-3] + + h, h_0, h_1 = lambda_t - lambda_s0, lambda_s0 - lambda_s1, lambda_s1 - lambda_s2 + r0, r1 = h_0 / h, h_1 / h + D0 = m0 + D1_0, D1_1 = (1.0 / r0) * (m0 - m1), (1.0 / r1) * (m1 - m2) + D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1) + D2 = (1.0 / (r0 + r1)) * (D1_0 - D1_1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + - (alpha_t * ((torch.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + - (sigma_t * ((torch.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2 + ) + return x_t + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.Tensor, + timestep: int, + sample: torch.Tensor, + generator=None, + variance_noise: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep DPMSolver. + + Args: + model_output (`torch.Tensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + variance_noise (`torch.Tensor`): + Alternative to generating noise with `generator` by directly providing the noise for the variance + itself. Useful for methods such as [`LEdits++`]. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Improve numerical stability for small number of steps + lower_order_final = (self.step_index == len(self.timesteps) - 1) and ( + self.config.euler_at_final + or (self.config.lower_order_final and len(self.timesteps) < 15) + or self.config.final_sigmas_type == "zero" + ) + lower_order_second = ( + (self.step_index == len(self.timesteps) - 2) and self.config.lower_order_final and len(self.timesteps) < 15 + ) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"] and variance_noise is None: + noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=torch.float32 + ) + elif self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]: + noise = variance_noise.to(device=model_output.device, dtype=torch.float32) + else: + noise = None + + if self.config.solver_order == 1 or self.lower_order_nums < 1 or lower_order_final: + prev_sample = self.dpm_solver_first_order_update(model_output, sample=sample, noise=noise) + elif self.config.solver_order == 2 or self.lower_order_nums < 2 or lower_order_second: + prev_sample = self.multistep_dpm_solver_second_order_update(self.model_outputs, sample=sample, noise=noise) + else: + prev_sample = self.multistep_dpm_solver_third_order_update(self.model_outputs, sample=sample) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # Cast sample back to expected dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + # alpha_t = self.alpha_t.to(device=original_samples.device, dtype=original_samples.dtype) + # sigma_t = self.sigma_t.to(device=original_samples.device, dtype=original_samples.dtype) + alpha_t = self.alpha_t.to(original_samples.device).to(original_samples.dtype) + sigma_t = self.sigma_t.to(original_samples.device).to(original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + alpha_t = alpha_t[timesteps].flatten() + while len(alpha_t.shape) < len(original_samples.shape): + alpha_t = alpha_t.unsqueeze(-1) + + sigma_t = sigma_t[timesteps].flatten() + while len(sigma_t.shape) < len(original_samples.shape): + sigma_t = sigma_t.unsqueeze(-1) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def get_velocity(self, original_samples: torch.Tensor, noise: torch.Tensor, timesteps: torch.IntTensor) -> torch.Tensor: + # alpha_t = self.alpha_t.to(device=original_samples.device, dtype=original_samples.dtype) + # sigma_t = self.sigma_t.to(device=original_samples.device, dtype=original_samples.dtype) + alpha_t = self.alpha_t.to(original_samples.device).to(original_samples.dtype) + sigma_t = self.sigma_t.to(original_samples.device).to(original_samples.dtype) + + timesteps = timesteps.to(original_samples.device) + alpha_t = alpha_t[timesteps].flatten() + while len(alpha_t.shape) < len(original_samples.shape): + alpha_t = alpha_t.unsqueeze(-1) + + sigma_t = sigma_t[timesteps].flatten() + while len(sigma_t.shape) < len(original_samples.shape): + sigma_t = sigma_t.unsqueeze(-1) + + velocity = alpha_t * noise - sigma_t * original_samples + return velocity + + def __len__(self): + return self.config.num_train_timesteps \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/schedule/timestep_sampler.py b/VibeVoice-ComfyUI/vvembed/schedule/timestep_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..524ee7f188b12d791069779460e72af5a40a2e12 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/schedule/timestep_sampler.py @@ -0,0 +1,19 @@ +import math +import torch + + +class UniformSampler: + def __init__(self, timesteps = 1000): + self.timesteps = timesteps + def sample(self, batch_size, device): + return torch.randint(0, self.timesteps, (batch_size,), device=device) + +class LogitNormalSampler: + def __init__(self, timesteps = 1000, m = 0, s = 1): + self.timesteps = timesteps + timesteps = torch.linspace(0, 1, timesteps) + logit = torch.log(timesteps / (1 - timesteps)) + self.prob = torch.exp(-0.5 * (logit - m) ** 2 / s ** 2) / (s * math.sqrt(2 * math.pi)) + def sample(self, batch_size, device): + return torch.multinomial(self.prob, batch_size, replacement=True).to(device) + \ No newline at end of file diff --git a/VibeVoice-ComfyUI/vvembed/scripts/__init__.py b/VibeVoice-ComfyUI/vvembed/scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VibeVoice-ComfyUI/vvembed/scripts/convert_nnscaler_checkpoint_to_transformers.py b/VibeVoice-ComfyUI/vvembed/scripts/convert_nnscaler_checkpoint_to_transformers.py new file mode 100644 index 0000000000000000000000000000000000000000..229f4eab5189240df2663257f908cdc904429080 --- /dev/null +++ b/VibeVoice-ComfyUI/vvembed/scripts/convert_nnscaler_checkpoint_to_transformers.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# coding=utf-8 + +import argparse +import json +import os +from pathlib import Path +import re +import torch +from typing import Dict, List, Tuple + +from modular.configuration_vibevoice import ( + VibeVoiceConfig +) +from modular.modeling_vibevoice import VibeVoiceForConditionalGeneration +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +def convert_vibevoice_nnscaler_checkpoint_to_hf( + checkpoint_path: str, + pytorch_dump_folder_path: str, + config_path: str = None, +): + """ + Convert a nnscaler VibeVoice checkpoint to HuggingFace format. + Supports both regular checkpoints and tensor parallel checkpoints. + """ + + # Load regular checkpoint + logger.info(f"Loading regular checkpoint from {checkpoint_path}") + checkpoint = torch.load(checkpoint_path, map_location="cpu") # ['model', 'optimizer', 'lr_scheduler', 'train_status', 'train_args', 'rng_states', 'nnscaler', 'dataloader'] + + # config = checkpoint['train_args'] + init_config_name = checkpoint['train_args']['vars']['model_args']['config_path']['relative_path'] + pretrained_name = checkpoint['train_args']['vars']['data_args']['tokenizer_path'] + + init_config_path = Path(__file__).parent.parent / 'configs' / init_config_name.split('/')[-1] + if init_config_path.exists(): + logger.info(f"Loading initial config from {init_config_path}") + with open(init_config_path, 'r') as f: + init_config = json.load(f) + else: + raise FileNotFoundError(f"Initial config file {init_config_path} not found. Please provide a valid path.") + + tie_word_embeddings = init_config['decoder_config'].get('tie_word_embeddings', True) + logger.info(f"Tie word embeddings: {tie_word_embeddings}") + + init_config['decoder_config']['use_cache'] = True + config = VibeVoiceConfig(**init_config, tie_word_embeddings=tie_word_embeddings) + + # # Extract the model state dict + model_state_dict = {k.replace('model.model.', 'model.'): v for k, v in checkpoint["model"].items() if k.startswith('model.model.')} + if not tie_word_embeddings and 'model.lm_head.weight' in checkpoint["model"].keys(): + # If not tying weights, we need to add the lm_head weight separately + model_state_dict['lm_head.weight'] = checkpoint["model"]['model.lm_head.weight'] + + # Override with provided config if available + if config_path: + logger.info(f"Loading config from {config_path}") + with open(config_path, 'r') as f: + config_dict = json.load(f) + config = VibeVoiceConfig.from_dict(config_dict) + + # Set the default dtype to bfloat16 before creating the model + original_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.bfloat16) + + # Create the HuggingFace model + logger.info("Creating HuggingFace VibeVoiceForConditionalGeneration model") + model = VibeVoiceForConditionalGeneration(config) + + # Restore original dtype + torch.set_default_dtype(original_dtype) + + # Load the state dict + logger.info("Loading weights into model") + missing_keys, unexpected_keys = model.load_state_dict(model_state_dict, strict=False) + + if missing_keys: + logger.warning(f"Missing keys: {missing_keys}") + if unexpected_keys: + logger.warning(f"Unexpected keys: {unexpected_keys}") + + # Create output directory + os.makedirs(pytorch_dump_folder_path, exist_ok=True) + + # Save the model and config + logger.info(f"Saving model to {pytorch_dump_folder_path}") + + # Save config + config.save_pretrained(pytorch_dump_folder_path) + + # Save VibeVoiceProcessor configuration + logger.info("Saving VibeVoiceProcessor configuration") + processor_config = { + "processor_class": "VibeVoiceProcessor", + "speech_tok_compress_ratio": 3200, + "db_normalize": True, + # Audio processor configuration + "audio_processor": { + "feature_extractor_type": "VibeVoiceTokenizerProcessor", + "sampling_rate": 24000, + "normalize_audio": True, + "target_dB_FS": -25, + "eps": 1e-6, + }, + "language_model_pretrained_name": pretrained_name, + } + + processor_config_path = os.path.join(pytorch_dump_folder_path, "preprocessor_config.json") + with open(processor_config_path, 'w') as f: + json.dump(processor_config, f, indent=2) + logger.info(f"Saved processor config to {processor_config_path}") + + # Save model with sharding + # save_pretrained handles tied weights automatically + logger.info("Saving model weights with sharding...") + model.save_pretrained( + pytorch_dump_folder_path, + max_shard_size="2GB", # Set maximum size for each shard + safe_serialization=True # Ensure saving in .safetensors format + ) + logger.info(f"Model weights saved to {pytorch_dump_folder_path}") + + logger.info("Conversion complete!") + + # Verify the saved model can be loaded + logger.info("Verifying saved model...") + loaded_model = VibeVoiceForConditionalGeneration.from_pretrained(pytorch_dump_folder_path) + logger.info("Model successfully loaded from saved checkpoint!") + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--nnscaler_checkpoint_path", + type=str, + required=True, + help="Path to the fairseq checkpoint (.pt file). For tensor parallel checkpoints, " + "provide any one of the part files (e.g., checkpoint_1_5000-model_part-0.pt), " + "and the script will automatically detect and merge all parts.", + ) + parser.add_argument( + "--pytorch_dump_folder_path", + type=str, + required=True, + help="Path to the output PyTorch model directory", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help="Optional path to a config JSON file to override extracted config", + ) + + args = parser.parse_args() + + convert_vibevoice_nnscaler_checkpoint_to_hf( + args.nnscaler_checkpoint_path, + args.pytorch_dump_folder_path, + args.config_path, + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/comfyui-qwen-asr/.github/workflows/publish.yml b/comfyui-qwen-asr/.github/workflows/publish.yml new file mode 100644 index 0000000000000000000000000000000000000000..122a1aeb5c04122e7f252c6d18bac91bfaf86fb5 --- /dev/null +++ b/comfyui-qwen-asr/.github/workflows/publish.yml @@ -0,0 +1,20 @@ +name: Publish to Comfy registry +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "pyproject.toml" + +jobs: + publish-node: + name: Publish Custom Node to registry + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Publish Custom Node + uses: Comfy-Org/publish-node-action@main + with: + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} ## Add your own personal access token to your Github Repository secrets and reference it here. \ No newline at end of file diff --git a/comfyui-qwen-asr/.gitignore b/comfyui-qwen-asr/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..9bc3ee609a180c5b7815270185a145c2ea059af6 --- /dev/null +++ b/comfyui-qwen-asr/.gitignore @@ -0,0 +1,29 @@ +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +.env +.venv +env/ +venv/ +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store diff --git a/comfyui-qwen-asr/.tracking b/comfyui-qwen-asr/.tracking new file mode 100644 index 0000000000000000000000000000000000000000..6c3994aa9762be4bc1450bd6f041ce9bc9ca36d2 --- /dev/null +++ b/comfyui-qwen-asr/.tracking @@ -0,0 +1,9 @@ +.github/workflows/publish.yml +.gitignore +README.md +__init__.py +assets/intro.png +example_workflows/base.json +nodes.py +pyproject.toml +requirements.txt \ No newline at end of file diff --git a/comfyui-qwen-asr/README.md b/comfyui-qwen-asr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fc9bdad359a76a82890232ecd36d2cfa7dee7da0 --- /dev/null +++ b/comfyui-qwen-asr/README.md @@ -0,0 +1,92 @@ +# ComfyUI-Qwen3-ASR + +ComfyUI custom nodes for **Qwen3-ASR** (Automatic Speech Recognition) - audio-to-text transcription supporting 52 languages and dialects. + +> 🔗 Compatible with [ComfyUI-Qwen3-TTS](https://github.com/DarioFT/ComfyUI-Qwen3-TTS) for complete speech workflows + +## Features + +- **Multi-language**: 30 languages + 22 Chinese dialects +- **Two model sizes**: 1.7B (best quality) and 0.6B (faster) +- **Auto language detection**: No need to specify language +- **Timestamps**: Optional word/character-level timing via Forced Aligner +- **Batch processing**: Transcribe multiple audio files +- **Auto-download**: Models download automatically on first use + +## Installation + +### Via ComfyUI Manager (Recommended) +Search for "Qwen3-ASR" in ComfyUI Manager + +### Manual Installation +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/DarioFT/ComfyUI-Qwen3-ASR.git +cd ComfyUI-Qwen3-ASR +pip install -r requirements.txt +``` + +## Nodes + +### Qwen3-ASR Loader +Loads the ASR model with auto-download support. + +| Input | Type | Description | +|-------|------|-------------| +| repo_id | dropdown | Model: 1.7B or 0.6B | +| source | dropdown | HuggingFace or ModelScope | +| precision | dropdown | fp16, bf16, fp32 | +| attention | dropdown | auto, flash_attention_2, sdpa, eager | +| forced_aligner | dropdown | Optional aligner for timestamps | +| local_model_path | string | Optional custom model path | + +### Qwen3-ASR Transcribe +Transcribes a single audio input to text. + +| Input | Type | Description | +|-------|------|-------------| +| model | QWEN3_ASR_MODEL | Loaded model | +| audio | AUDIO | Audio input (ComfyUI format) | +| language | dropdown | Force language or "auto" | +| context | string | Optional context hints | +| return_timestamps | boolean | Enable timestamp output | + +| Output | Type | Description | +|--------|------|-------------| +| text | STRING | Transcribed text | +| language | STRING | Detected language | +| timestamps | STRING | Word-level timestamps (if enabled) | + +### Qwen3-ASR Batch Transcribe +Batch transcription for multiple audio files. + +## Supported Languages + +Chinese, English, Cantonese, Arabic, German, French, Spanish, Portuguese, Indonesian, Italian, Korean, Russian, Thai, Vietnamese, Japanese, Turkish, Hindi, Malay, Dutch, Swedish, Danish, Finnish, Polish, Czech, Filipino, Persian, Greek, Hungarian, Macedonian, Romanian + +Plus 22 Chinese dialects including Sichuan, Cantonese (HK/Guangdong), Wu, Minnan, and regional accents. + +## Workflow Examples + +### Basic Transcription +``` +LoadAudio → Qwen3-ASR Loader → Qwen3-ASR Transcribe → ShowText +``` + +### With TTS (Speech-to-Speech) +``` +LoadAudio → Qwen3-ASR Transcribe → [process text] → Qwen3-TTS → SaveAudio +``` + +## Model Storage + +Models are stored in: `ComfyUI/models/Qwen3-ASR/` + +## Credits + +- [Qwen3-ASR](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) by Alibaba Qwen Team +- [qwen-asr](https://pypi.org/project/qwen-asr/) Python package + +## License + +Apache-2.0 diff --git a/comfyui-qwen-asr/__init__.py b/comfyui-qwen-asr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4b496c5dd387a69f1aab0a34cfab5dd6dee7fa57 --- /dev/null +++ b/comfyui-qwen-asr/__init__.py @@ -0,0 +1,19 @@ +from .nodes import ( + Qwen3ASRLoader, + Qwen3ASRTranscribe, + Qwen3ASRBatchTranscribe, +) + +NODE_CLASS_MAPPINGS = { + "Qwen3ASRLoader": Qwen3ASRLoader, + "Qwen3ASRTranscribe": Qwen3ASRTranscribe, + "Qwen3ASRBatchTranscribe": Qwen3ASRBatchTranscribe, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "Qwen3ASRLoader": "Qwen3-ASR Loader", + "Qwen3ASRTranscribe": "Qwen3-ASR Transcribe", + "Qwen3ASRBatchTranscribe": "Qwen3-ASR Batch Transcribe", +} + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/comfyui-qwen-asr/__pycache__/__init__.cpython-312.pyc b/comfyui-qwen-asr/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..136b3354dfb7139768011a1c612b8f847ebf50c8 Binary files /dev/null and b/comfyui-qwen-asr/__pycache__/__init__.cpython-312.pyc differ diff --git a/comfyui-qwen-asr/__pycache__/nodes.cpython-312.pyc b/comfyui-qwen-asr/__pycache__/nodes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b533f925711d51be324b7573f59d1fc99e685e0c Binary files /dev/null and b/comfyui-qwen-asr/__pycache__/nodes.cpython-312.pyc differ diff --git a/comfyui-qwen-asr/assets/intro.png b/comfyui-qwen-asr/assets/intro.png new file mode 100644 index 0000000000000000000000000000000000000000..861d4e5627a64436cfa3e96159eeaa50249a7854 --- /dev/null +++ b/comfyui-qwen-asr/assets/intro.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf528a961ff1f3929d7f10a65ab92501d49143fa2c0bfb70c06ebf4dc0b79a5c +size 147936 diff --git a/comfyui-qwen-asr/example_workflows/base.json b/comfyui-qwen-asr/example_workflows/base.json new file mode 100644 index 0000000000000000000000000000000000000000..0d71ad71941dab99cbc75f58cf60ce7aac608587 --- /dev/null +++ b/comfyui-qwen-asr/example_workflows/base.json @@ -0,0 +1,207 @@ +{ + "id": "560c0123-a3c9-4148-b0cb-7b705dd02044", + "revision": 0, + "last_node_id": 5, + "last_link_id": 3, + "nodes": [ + { + "id": 4, + "type": "LoadAudio", + "pos": [ + 26.0944883333357, + 248.1008013888877 + ], + "size": [ + 282.83333587646484, + 136 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 1 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.10.0", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "1.wav", + null, + "" + ] + }, + { + "id": 1, + "type": "Qwen3ASRLoader", + "pos": [ + 31.015860833335807, + -4.379546388890003 + ], + "size": [ + 270, + 178 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "QWEN3_ASR_MODEL", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "Qwen3ASRLoader" + }, + "widgets_values": [ + "Qwen/Qwen3-ASR-1.7B", + "HuggingFace", + "bf16", + "auto", + "None", + "" + ] + }, + { + "id": 2, + "type": "Qwen3ASRTranscribe", + "pos": [ + 366.6877755555577, + 51.57828166666546 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "QWEN3_ASR_MODEL", + "link": 2 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 1 + } + ], + "outputs": [ + { + "name": "text", + "type": "STRING", + "links": [ + 3 + ] + }, + { + "name": "language", + "type": "STRING", + "links": null + }, + { + "name": "timestamps", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "Qwen3ASRTranscribe" + }, + "widgets_values": [ + "auto", + "", + false + ] + }, + { + "id": 5, + "type": "PreviewAny", + "pos": [ + 837.7456827777796, + 51.88921805555435 + ], + "size": [ + 210, + 166 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 3 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.10.0", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + false + ] + } + ], + "links": [ + [ + 1, + 4, + 0, + 2, + 1, + "AUDIO" + ], + [ + 2, + 1, + 0, + 2, + 0, + "QWEN3_ASR_MODEL" + ], + [ + 3, + 2, + 0, + 5, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "workflowRendererVersion": "LG", + "ue_links": [], + "ds": { + "scale": 1.128526645768025, + "offset": [ + 249.0430116666644, + 262.42325416666796 + ] + }, + "frontendVersion": "1.37.11" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui-qwen-asr/nodes.py b/comfyui-qwen-asr/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..8969a9c9d89e503d07bfbcf3828408c5eaef482e --- /dev/null +++ b/comfyui-qwen-asr/nodes.py @@ -0,0 +1,275 @@ +import os +import shutil +import torch +import numpy as np +import folder_paths +import comfy.model_management as mm +from qwen_asr import Qwen3ASRModel + +# Register Qwen3-ASR models folder with ComfyUI +QWEN3_ASR_MODELS_DIR = os.path.join(folder_paths.models_dir, "Qwen3-ASR") +os.makedirs(QWEN3_ASR_MODELS_DIR, exist_ok=True) +folder_paths.add_model_folder_path("Qwen3-ASR", QWEN3_ASR_MODELS_DIR) + +# Model repo mappings +QWEN3_ASR_MODELS = { + "Qwen/Qwen3-ASR-1.7B": "Qwen3-ASR-1.7B", + "Qwen/Qwen3-ASR-0.6B": "Qwen3-ASR-0.6B", +} + +QWEN3_FORCED_ALIGNERS = { + "None": None, + "Qwen/Qwen3-ForcedAligner-0.6B": "Qwen3-ForcedAligner-0.6B", +} + +# Supported languages +SUPPORTED_LANGUAGES = [ + "auto", + "Chinese", "English", "Cantonese", "Arabic", "German", "French", "Spanish", + "Portuguese", "Indonesian", "Italian", "Korean", "Russian", "Thai", + "Vietnamese", "Japanese", "Turkish", "Hindi", "Malay", "Dutch", "Swedish", + "Danish", "Finnish", "Polish", "Czech", "Filipino", "Persian", "Greek", + "Hungarian", "Macedonian", "Romanian" +] + + +def get_local_model_path(repo_id: str) -> str: + folder_name = QWEN3_ASR_MODELS.get(repo_id) or QWEN3_FORCED_ALIGNERS.get(repo_id) or repo_id.replace("/", "_") + return os.path.join(QWEN3_ASR_MODELS_DIR, folder_name) + + +def migrate_cached_model(repo_id: str, target_path: str) -> bool: + if os.path.exists(target_path) and os.listdir(target_path): + return True + + hf_cache = os.path.join(os.path.expanduser("~"), ".cache", "huggingface", "hub") + hf_model_dir = os.path.join(hf_cache, f"models--{repo_id.replace('/', '--')}") + if os.path.exists(hf_model_dir): + snapshots_dir = os.path.join(hf_model_dir, "snapshots") + if os.path.exists(snapshots_dir): + snapshots = os.listdir(snapshots_dir) + if snapshots: + source = os.path.join(snapshots_dir, snapshots[0]) + print(f"Migrating model from HuggingFace cache: {source} -> {target_path}") + shutil.copytree(source, target_path, dirs_exist_ok=True) + return True + + ms_cache = os.path.join(os.path.expanduser("~"), ".cache", "modelscope", "hub") + ms_model_dir = os.path.join(ms_cache, repo_id.replace("/", os.sep)) + if os.path.exists(ms_model_dir): + print(f"Migrating model from ModelScope cache: {ms_model_dir} -> {target_path}") + shutil.copytree(ms_model_dir, target_path, dirs_exist_ok=True) + return True + + return False + + +def download_model_to_comfyui(repo_id: str, source: str) -> str: + target_path = get_local_model_path(repo_id) + + if migrate_cached_model(repo_id, target_path): + print(f"Model available at: {target_path}") + return target_path + + os.makedirs(target_path, exist_ok=True) + + if source == "ModelScope": + from modelscope import snapshot_download + print(f"Downloading {repo_id} from ModelScope to {target_path}...") + snapshot_download(repo_id, local_dir=target_path) + else: + from huggingface_hub import snapshot_download + print(f"Downloading {repo_id} from HuggingFace to {target_path}...") + snapshot_download(repo_id, local_dir=target_path) + + return target_path + + +def load_audio_input(audio_input): + if audio_input is None: + return None + + waveform = audio_input["waveform"] + sr = audio_input["sample_rate"] + + wav = waveform[0] + + if wav.shape[0] > 1: + wav = torch.mean(wav, dim=0) + else: + wav = wav.squeeze(0) + + return (wav.numpy().astype(np.float32), sr) + + +class Qwen3ASRLoader: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "repo_id": (list(QWEN3_ASR_MODELS.keys()), {"default": "Qwen/Qwen3-ASR-1.7B"}), + "source": (["HuggingFace", "ModelScope"], {"default": "HuggingFace"}), + "precision": (["fp16", "bf16", "fp32"], {"default": "bf16"}), + "attention": (["auto", "flash_attention_2", "sdpa", "eager"], {"default": "auto"}), + }, + "optional": { + "forced_aligner": (list(QWEN3_FORCED_ALIGNERS.keys()), {"default": "None"}), + "local_model_path": ("STRING", {"default": "", "multiline": False}), + } + } + + RETURN_TYPES = ("QWEN3_ASR_MODEL",) + RETURN_NAMES = ("model",) + FUNCTION = "load_model" + CATEGORY = "Qwen3-ASR" + + def load_model(self, repo_id, source, precision, attention, forced_aligner="None", local_model_path=""): + device = mm.get_torch_device() + + dtype = torch.float32 + if precision == "bf16": + if device.type == "mps": + dtype = torch.float16 + print("Note: Using fp16 on MPS (bf16 has limited support)") + else: + dtype = torch.bfloat16 + elif precision == "fp16": + dtype = torch.float16 + + if local_model_path and local_model_path.strip() != "": + model_path = local_model_path.strip() + print(f"Loading from local path: {model_path}") + else: + local_path = get_local_model_path(repo_id) + if os.path.exists(local_path) and os.listdir(local_path): + model_path = local_path + print(f"Loading from ComfyUI models folder: {model_path}") + else: + model_path = download_model_to_comfyui(repo_id, source) + + model_kwargs = dict( + dtype=dtype, + device_map=str(device), + max_inference_batch_size=32, + max_new_tokens=256, + ) + if attention != "auto": + model_kwargs["attn_implementation"] = attention + + if forced_aligner and forced_aligner != "None": + aligner_local = get_local_model_path(forced_aligner) + if not (os.path.exists(aligner_local) and os.listdir(aligner_local)): + aligner_local = download_model_to_comfyui(forced_aligner, source) + model_kwargs["forced_aligner"] = aligner_local + model_kwargs["forced_aligner_kwargs"] = dict( + dtype=dtype, + device_map=str(device), + ) + if attention != "auto": + model_kwargs["forced_aligner_kwargs"]["attn_implementation"] = attention + + print(f"Loading Qwen3-ASR model from {model_path}...") + model = Qwen3ASRModel.from_pretrained(model_path, **model_kwargs) + + return (model,) + + +class Qwen3ASRTranscribe: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("QWEN3_ASR_MODEL",), + "audio": ("AUDIO",), + }, + "optional": { + "language": (SUPPORTED_LANGUAGES, {"default": "auto"}), + "context": ("STRING", {"default": "", "multiline": True}), + "return_timestamps": ("BOOLEAN", {"default": False}), + } + } + + RETURN_TYPES = ("STRING", "STRING", "STRING") + RETURN_NAMES = ("text", "language", "timestamps") + FUNCTION = "transcribe" + CATEGORY = "Qwen3-ASR" + + def transcribe(self, model, audio, language="auto", context="", return_timestamps=False): + audio_data = load_audio_input(audio) + if audio_data is None: + return ("", "", "") + + lang = None if language == "auto" else language + ctx = context if context.strip() else "" + + results = model.transcribe( + audio=audio_data, + language=lang, + context=ctx if ctx else None, + return_time_stamps=return_timestamps, + ) + + result = results[0] + text = result.text + detected_lang = result.language or "" + + timestamps_str = "" + if return_timestamps and result.time_stamps: + ts_lines = [] + for ts in result.time_stamps: + ts_lines.append(f"{ts.start_time:.2f}-{ts.end_time:.2f}: {ts.text}") + timestamps_str = "\n".join(ts_lines) + + return (text, detected_lang, timestamps_str) + + +class Qwen3ASRBatchTranscribe: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("QWEN3_ASR_MODEL",), + "audio_list": ("AUDIO",), + }, + "optional": { + "language": (SUPPORTED_LANGUAGES, {"default": "auto"}), + "return_timestamps": ("BOOLEAN", {"default": False}), + } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("transcriptions",) + FUNCTION = "batch_transcribe" + CATEGORY = "Qwen3-ASR" + + def batch_transcribe(self, model, audio_list, language="auto", return_timestamps=False): + if not isinstance(audio_list, list): + audio_list = [audio_list] + + audio_inputs = [] + for audio in audio_list: + audio_data = load_audio_input(audio) + if audio_data: + audio_inputs.append(audio_data) + + if not audio_inputs: + return ("",) + + lang = None if language == "auto" else language + languages = [lang] * len(audio_inputs) if lang else None + + results = model.transcribe( + audio=audio_inputs, + language=languages, + return_time_stamps=return_timestamps, + ) + + output_lines = [] + for i, result in enumerate(results): + line = f"[{i}] ({result.language}): {result.text}" + output_lines.append(line) + if return_timestamps and result.time_stamps: + for ts in result.time_stamps: + output_lines.append(f" {ts.start_time:.2f}-{ts.end_time:.2f}: {ts.text}") + + return ("\n".join(output_lines),) diff --git a/comfyui-qwen-asr/pyproject.toml b/comfyui-qwen-asr/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..edfc838c54a1a023426fee34c1be523ab43f4acf --- /dev/null +++ b/comfyui-qwen-asr/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "comfyui-qwen-asr" +description = "A ComfyUI custom node suite for Qwen3-ASR, supporting speech-to-text transcription with 1.7B and 0.6B models, 52 languages/dialects, and optional timestamp alignment." +version = "1.0.0" +license = { text = "Apache-2.0" } + +dependencies = ["qwen-asr", "modelscope", "soundfile", "numpy", "torch", "transformers", "accelerate"] + +[project.urls] +Repository = "https://github.com/DarioFT/ComfyUI-Qwen3-ASR" +Documentation = "https://github.com/DarioFT/ComfyUI-Qwen3-ASR/wiki" +"Bug Tracker" = "https://github.com/DarioFT/ComfyUI-Qwen3-ASR/issues" + +[tool.comfy] +PublisherId = "darioft" +DisplayName = "ComfyUI-Qwen3-ASR" diff --git a/comfyui-qwen-asr/requirements.txt b/comfyui-qwen-asr/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..fe134d5206279f9db7c4d782591f620293ec12db --- /dev/null +++ b/comfyui-qwen-asr/requirements.txt @@ -0,0 +1,7 @@ +qwen-asr +modelscope +soundfile +numpy +torch +transformers +accelerate diff --git a/comfyui-videohelpersuite/.github/workflows/publish.yml b/comfyui-videohelpersuite/.github/workflows/publish.yml new file mode 100644 index 0000000000000000000000000000000000000000..27bcdb11695ea19286da34cd5f36cf75155a4dbb --- /dev/null +++ b/comfyui-videohelpersuite/.github/workflows/publish.yml @@ -0,0 +1,24 @@ +name: Publish to Comfy registry +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "pyproject.toml" + +permissions: + issues: write + +jobs: + publish-node: + name: Publish Custom Node to registry + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'Kosinkadink' }} + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Publish Custom Node + uses: Comfy-Org/publish-node-action@v1 + with: + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} ## Add your own personal access token to your Github Repository secrets and reference it here. diff --git a/comfyui-videohelpersuite/.gitignore b/comfyui-videohelpersuite/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..9b5379e8f93da3909706b0a7d37f5c8ab5e2a2a5 --- /dev/null +++ b/comfyui-videohelpersuite/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ +# VIM swap files +*.swp diff --git a/comfyui-videohelpersuite/.tracking b/comfyui-videohelpersuite/.tracking new file mode 100644 index 0000000000000000000000000000000000000000..359ab1f5ce25b8a8f789bcb6a70c0d776b139643 --- /dev/null +++ b/comfyui-videohelpersuite/.tracking @@ -0,0 +1,45 @@ +.github/workflows/publish.yml +.gitignore +LICENSE +README.md +__init__.py +pyproject.toml +requirements.txt +testframework/README.md +testframework/__init__.py +testframework/server.py +testframework/web/js/testRunner.js +tests/README.md +tests/audio.json +tests/batch4x4.json +tests/converted-format-input.json +tests/converted-input.json +tests/loop.json +tests/old-prores.json +tests/old-vae-conversion.json +tests/simple.json +video_formats/16bit-png.json +video_formats/8bit-png.json +video_formats/ProRes.json +video_formats/av1-webm.json +video_formats/ffmpeg-gif.json +video_formats/ffv1-mkv.json +video_formats/gifski.json +video_formats/h264-mp4.json +video_formats/h265-mp4.json +video_formats/nvenc_av1-mp4.json +video_formats/nvenc_h264-mp4.json +video_formats/nvenc_hevc-mp4.json +video_formats/webm.json +videohelpersuite/batched_nodes.py +videohelpersuite/documentation.py +videohelpersuite/image_latent_nodes.py +videohelpersuite/latent_preview.py +videohelpersuite/load_images_nodes.py +videohelpersuite/load_video_nodes.py +videohelpersuite/logger.py +videohelpersuite/nodes.py +videohelpersuite/server.py +videohelpersuite/utils.py +web/js/VHS.core.js +web/js/videoinfo.js \ No newline at end of file diff --git a/comfyui-videohelpersuite/LICENSE b/comfyui-videohelpersuite/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f288702d2fa16d3cdf0035b15a9fcbc552cd88e7 --- /dev/null +++ b/comfyui-videohelpersuite/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/comfyui-videohelpersuite/README.md b/comfyui-videohelpersuite/README.md new file mode 100644 index 0000000000000000000000000000000000000000..753fcecf9979b40b2427869c6ad5bed1b61df482 --- /dev/null +++ b/comfyui-videohelpersuite/README.md @@ -0,0 +1,110 @@ +# ComfyUI-VideoHelperSuite +Nodes related to video workflows + +## I/O Nodes +### Load Video +Converts a video file into a series of images +- video: The video file to be loaded +- force_rate: Discards or duplicates frames as needed to hit a target frame rate. Disabled by setting to 0. This can be used to quickly match a suggested frame rate like the 8 fps of AnimateDiff. +- force_size: Allows for quick resizing to a number of suggested sizes. Several options allow you to set only width or height and determine the other from aspect ratio. +- frame_load_cap: The maximum number of frames which will be returned. This could also be thought of as the maximum batch size. +- skip_first_frames: How many frames to skip from the start of the video after adjusting for a forced frame rate. By incrementing this number by the frame_load_cap, you can easily process a longer input video in parts. +- select_every_nth: Allows for skipping a number of frames without considering the base frame rate or risking frame duplication. Often useful when working with animated gifs +A path variant of the Load Video node exists that allows loading videos from external paths +![step](https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite/assets/4284322/b5fc993c-5c9b-4608-afa4-48ae2e1380ef) +![resize](https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite/assets/4284322/98d2e78e-1c44-443c-a8fe-0dab0b5947f3) +If [Advanced Previews](#advanced-previews) is enabled in the options menu of the web ui, the preview will reflect the current settings on the node. +### Load Image Sequence +Loads all image files from a subfolder. Options are similar to Load Video. +- image_load_cap: The maximum number of images which will be returned. This could also be thought of as the maximum batch size. +- skip_first_images: How many images to skip. By incrementing this number by image_load_cap, you can easily divide a long sequence of images into multiple batches. +- select_every_nth: Allows for skipping a number of images between every returned frame. + +A path variant of Load Image sequence also exists. +### Video Combine +Combines a series of images into an output video +If the optional audio input is provided, it will also be combined into the output video +- frame_rate: How many of the input frames are displayed per second. A higher frame rate means that the output video plays faster and has less duration. This should usually be kept to 8 for AnimateDiff, or matched to the force_rate of a Load Video node. +- loop_count: How many additional times the video should repeat +- filename_prefix: The base file name used for output. + - You can save output to a subfolder: `subfolder/video` + - Like the builtin Save Image node, you can add timestamps. `%date:yyyy-MM-ddThh:mm:ss%` might become 2023-10-31T6:45:25 +- format: The file format to use. Advanced information on configuring or adding additional video formats can be found in the [Video Formats](#video-formats) section. +- pingpong: Causes the input to be played back in the reverse to create a clean loop. +- save_output: Whether the image should be put into the output directory or the temp directory. +Returns: a `VHS_FILENAMES` which consists of a boolean indicating if save_output is enabled and a list of the full filepaths of all generated outputs in the order created. Accordingly `output[1][-1]` will be the most complete output. + +Depending on the format chosen, additional options may become available, including +- crf: Describes the quality of the output video. A lower number gives a higher quality video and a larger file size, while a higher number gives a lower quality video with a smaller size. Scaling varies by codec, but visually lossless output generally occurs around 20. +- save_metadata: Includes a copy of the workflow in the output video which can be loaded by dragging and dropping the video, just like with images. +- pix_fmt: Changes how the pixel data is stored. `yuv420p10le` has higher color quality, but won't work on all devices +### Load Audio +Provides a way to load standalone audio files. +- seek_seconds: An optional start time for the audio file in seconds. + +## Latent/Image Nodes +A number of utility nodes exist for managing latents. For each, there is an equivalent node which works on images. +### Split Batch +Divides the latents into two sets. The first `split_index` latents go to output A and the remainder to output B. If less then `split_index` latents are provided as input, all are passed to output A and output B is empty. +### Merge Batch +Combines two groups of latents into a single output. The order of the output is the latents in A followed by the latents in B. +If the input groups are not the same size, the node provides options for rescaling the latents before merging. +### Select Every Nth +The first of every `select_every_nth` input is passed and the remainder are discarded +### Get Count +### Duplicate Batch + +## Video Previews +Load Video (Upload), Load Video (Path), Load Images (Upload), Load Images (Path) and Video Combine provide animated previews. +Nodes with previews provide additional functionality when right clicked +- Open preview +- Save preview +- Pause preview: Can improve performance with very large videos +- Hide preview: Can improve performance, save space +- Sync preview: Restarts all previews for side-by-side comparisons + +### Advanced Previews +Advanced Previews must be manually enabled by clicking the settings gear next to Queue Prompt and checking the box for VHS Advanced Previews. +If enabled, videos which are displayed in the ui will be converted with ffmpeg on request. This has several benefits +- Previews for Load Video nodes will reflect the settings on the node such as skip_first_frames and frame_load_cap + - This makes it easy to select an exact portion of an input video and sync it with outputs +- It can use substantially less bandwidth if running the server remotely +- It can greatly improve the browser performance by downsizing videos to the in ui resolution, particularly useful with animated gifs +- It allows for previews of videos that would not normally be playable in browser. +- Can be limited to subdirectories of ComyUI if `VHS_STRICT_PATHS` is set as an environment variable. + +This fucntionality is disabled since it comes with several downsides +- There is a delay before videos show in the browser. This delay can become quite large if the input video is long +- The preview videos are lower quality (The original can always be viewed with Right Click -> Open preview) + +## Video Formats +Those familiar with ffmpeg are able to add json files to the video_formats folders to add new output types to Video Combine. +Consider the following example for av1-webm +```json +{ + "main_pass": + [ + "-n", "-c:v", "libsvtav1", + "-pix_fmt", "yuv420p10le", + "-crf", ["crf","INT", {"default": 23, "min": 0, "max": 100, "step": 1}] + ], + "audio_pass": ["-c:a", "libopus"], + "extension": "webm", + "environment": {"SVT_LOG": "1"} +} +``` +Most configuration takes place in `main_pass`, which is a list of arguments that are passed to ffmpeg. +- `"-n"` designates that the command should fail if a file of the same name already exists. This should never happen, but if some bug were to occur, it would ensure other files aren't overwritten. +- `"-c:v", "libsvtav1"` designates that the video should be encoded with an av1 codec using the new SVT-AV1 encoder. SVT-AV1 is much faster than libaom-av1, but may not exist in older versions of ffmpeg. Alternatively, av1_nvenc could be used for gpu encoding with newer nvidia cards. +- `"-pix_fmt", "yuv420p10le"` designates the standard pixel format with 10-bit color. It's important that some pixel format be specified to ensure a nonconfigurable input pix_fmt isn't used. + +`audio pass` contains a list of arguments which are passed to ffmpeg when audio is passed into Video Combine + +`extension` designates both the file extension and the container format that is used. If some of the above options are omitted from `main_pass` it can affect what default options are chosen. +`environment` can optionally be provided to set environment variables during execution. For av1 it's used to reduce the verbosity of logging so that only major errors are displayed. +`input_color_depth` effects the format in which pixels are passed to the ffmpeg subprocess. Current valid options are `8bit` and `16bit`. The later will produce higher quality output, but is experimental. + +Fields can be exposed in the webui as a widget using a format similar to what is used in the creation of custom nodes. In the above example, the argument for `-crf` will be exposed as a format widget in the webui. Format widgets are a list of up to 3 terms +- The name of the widget that will be displayed in the web ui +- Either a primitive such as "INT" or "BOOLEAN", or a list of string options +- A dictionary of options diff --git a/comfyui-videohelpersuite/__init__.py b/comfyui-videohelpersuite/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9fa995f0ac7e296a62bcba9df68b5250a0f7d041 --- /dev/null +++ b/comfyui-videohelpersuite/__init__.py @@ -0,0 +1,9 @@ +from .videohelpersuite.nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS +import folder_paths +from .videohelpersuite.server import server +from .videohelpersuite import documentation +from .videohelpersuite import latent_preview + +WEB_DIRECTORY = "./web" +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] +documentation.format_descriptions(NODE_CLASS_MAPPINGS) diff --git a/comfyui-videohelpersuite/__pycache__/__init__.cpython-312.pyc b/comfyui-videohelpersuite/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74df78c201ba38799fd9756d981cc38ccfd6747c Binary files /dev/null and b/comfyui-videohelpersuite/__pycache__/__init__.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/pyproject.toml b/comfyui-videohelpersuite/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..f3f4812b0fbca8bbf67129a6070beef6eb55fce8 --- /dev/null +++ b/comfyui-videohelpersuite/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "comfyui-videohelpersuite" +description = "Nodes related to video workflows" +version = "1.7.8" +license = { file = "LICENSE" } +dependencies = ["opencv-python", "imageio-ffmpeg"] + +[project.urls] +Repository = "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite" + +# Used by Comfy Registry https://comfyregistry.org +[tool.comfy] +PublisherId = "kosinkadink" +DisplayName = "ComfyUI-VideoHelperSuite" +Icon = "" diff --git a/comfyui-videohelpersuite/requirements.txt b/comfyui-videohelpersuite/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4fa34aa21b85c4b974e2a2b6891eae5fd6dd4164 --- /dev/null +++ b/comfyui-videohelpersuite/requirements.txt @@ -0,0 +1,2 @@ +opencv-python +imageio-ffmpeg diff --git a/comfyui-videohelpersuite/testframework/README.md b/comfyui-videohelpersuite/testframework/README.md new file mode 100644 index 0000000000000000000000000000000000000000..93cd503563080c9c7d785d419bef7dddc4c11cd8 --- /dev/null +++ b/comfyui-videohelpersuite/testframework/README.md @@ -0,0 +1,5 @@ +Code to automate execution of the tests and evaluate the results. +Distributed as a `custom node`, and can be installed by copying or simlinking to the `custom_nodes` directory. +Requires that ffprobe be available and added to the path. Note that imageio-ffmpeg does not bundle ffprobe. + +When installed, it adds a new sidebar tab to automate running one, or a folder of tests. This requires that the `Use new menu and workflow management` setting not be disabled diff --git a/comfyui-videohelpersuite/testframework/__init__.py b/comfyui-videohelpersuite/testframework/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c75b11c164596ef873898dd843700c7164cc0932 --- /dev/null +++ b/comfyui-videohelpersuite/testframework/__init__.py @@ -0,0 +1,6 @@ +from . import server +NODE_CLASS_MAPPINGS = {} +NODE_DISPLAY_NAME_MAPPINGS = {} + +WEB_DIRECTORY = "./web" +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/comfyui-videohelpersuite/testframework/server.py b/comfyui-videohelpersuite/testframework/server.py new file mode 100644 index 0000000000000000000000000000000000000000..a9c20ea3eab31bab288d6089a89410ff409d43ad --- /dev/null +++ b/comfyui-videohelpersuite/testframework/server.py @@ -0,0 +1,60 @@ +import subprocess +import json +import os +import torch +import shutil + +import server +import folder_paths + +web = server.web + +@server.PromptServer.instance.routes.post("/VHS_test") +async def test(request): + try: + req_data = await request.json() + output = req_data['output']['gifs'][0] + filename = output['filename'] + typ = output['type'] + base_args = ["ffprobe", "-v", "error", '-count_packets', "-show_entries", "stream", "-of", "json"] + video = folder_paths.get_annotated_filepath(f'{filename} [{typ}]') + vprobe = json.loads(subprocess.run(base_args + ['-select_streams', 'v:0', video], + capture_output=True, check=True).stdout)['streams'][0] + aprobe = json.loads(subprocess.run(base_args + ['-select_streams', 'a:0', video], + capture_output=True, check=True).stdout)['streams'] + probe = {'video': vprobe} + if len(aprobe) > 0: + probe['audio'] = aprobe[0] + errors = [] + compare = None + for test in req_data['tests']: + if test['type'] == 'compare': + compare = test + continue + key = test['key'] + expected = test['value'] + actual = probe[test['type']][key] + if expected != actual: + #Consider always dumping type? + errors.append(f'{key}: {expected} != {actual}') + if len(errors) == 0 and compare is not None: + if not os.path.exists(compare['filename']): + os.makedirs(os.path.split(compare['filename'])[0], exist_ok=True) + shutil.copy(video, compare['filename']) + print("Missing comparison file has been initialized from output:", os.path.abspath(compare['filename'])) + else: + #NOTE: This does not include the full memory optimizations of VHS + #Tests should be small + #TODO: Figure out way to do opacity comparison. May need to do blending in python + #(easy, but slower and more memory intensive) + diff = subprocess.run(['ffmpeg', '-v', 'error', '-i', video, '-i', compare['filename'], '-filter_complex', 'blend=all_mode=grainextract', '-pix_fmt', 'rgb24', '-f', 'rawvideo', '-'], stdout=subprocess.PIPE, check=True).stdout + diff = torch.frombuffer(diff, dtype=torch.uint8).to(dtype=torch.float32).div_(255) + #diff = diff.reshape((-1,4)) + d = (diff-0.5).abs().sum()/diff.size(0) + if d > compare['tolerance']: + errors.append(f'Similarity is outside specified tolerance: {d}') + else: + print('d:', d) + return web.json_response(errors) + except Exception as e: + return web.json_response(str(e)) diff --git a/comfyui-videohelpersuite/testframework/web/js/testRunner.js b/comfyui-videohelpersuite/testframework/web/js/testRunner.js new file mode 100644 index 0000000000000000000000000000000000000000..521a2204b9a2a3202441818461231872ea8a15c8 --- /dev/null +++ b/comfyui-videohelpersuite/testframework/web/js/testRunner.js @@ -0,0 +1,96 @@ + +import {app} from "../../../scripts/app.js"; +import {api} from "../../../scripts/api.js"; + +let watched_nodes = {} +let resolve = undefined +let testURL = api.apiURL("/VHS_test") +let errors = [] +api.addEventListener("executed", async function ({detail}) { + if (watched_nodes && watched_nodes[detail?.node]) { + if (detail?.output?.unfinished_batch) { + return + } + let requestBody = {tests: watched_nodes[detail.node], output: detail.output} + try { + let req = await fetch(api.apiURL("/VHS_test"), + {method: "POST", body: JSON.stringify(requestBody)}); + let testResult = await req.json() + if (testResult.length != 0) { + errors.push(testResult) + } + } catch(e) { + errors.push(e) + } + if (!(watched_nodes.length -= 1)) { + resolve() + } + } +}); + +const workflowService = app.extensionManager.workflow + +async function runTest(file) { + if (!file?.name?.endsWith(".json")) { + return false + } + let workflow = JSON.parse(await file.text()) + await app.loadGraphData(workflow) + //NOTE: API is not used so workflow data is actually processed + watched_nodes = workflow.tests + errors = [] + let p = new Promise((r) => resolve = r) + await app.queuePrompt() + //block until execution completes + await p + watched_nodes = {} + if (errors.length > 0) { + app.ui.dialog.show("Failed " + errors.length + " tests:\n" + errors) + return true + } + await workflowService.closeWorkflow(workflowService.activeWorkflow, {warnIfUnsaved: false}) + return false +} +let iconOverride = document.createElement("style") +iconOverride.innerHTML = `.VHSTestIcon:before {content: '🧪';}` +document.body.append(iconOverride) + +let testSidebar = {id: 'VHStest', title: 'VHS Test', icon: 'VHSTestIcon', type: 'custom', + render: (e) => { + e.innerHTML = `Select a folder containing tests + + Or select a single test + + ` + + const folderInput = e.children[0] + const fileInput = e.children[1] + Object.assign(folderInput, { + type: "file", + webkitdirectory: true, + onchange: async function() { + const startTime = Date.now() + let failedTests = false + for(const file of this.files) { + failedTests ||= await runTest(file) + } + this.value="" + if (!failedTests) { + console.log("All tests passed in " + ((Date.now() - startTime)/1000) + "s") + } + }, + }); + Object.assign(fileInput, { + type: "file", + accept: ".json", + onchange: async function() { + if (this.files.length) { + if(!(await runTest(this.files[0]))) { + console.log("Test complete") + } + this.value="" + } + }, + }); + }} +app.extensionManager.registerSidebarTab(testSidebar) diff --git a/comfyui-videohelpersuite/tests/README.md b/comfyui-videohelpersuite/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ec025a7d169c7783f34523c63a0a6ae9a2e00fdb --- /dev/null +++ b/comfyui-videohelpersuite/tests/README.md @@ -0,0 +1 @@ +Workflows for automated testing of VHS. Most include an additional tests key to check the properties or perform comparisons on node outputs diff --git a/comfyui-videohelpersuite/tests/audio.json b/comfyui-videohelpersuite/tests/audio.json new file mode 100644 index 0000000000000000000000000000000000000000..cae3d766731f7fe3ae4866f6fa8c0cc86a2ca46d --- /dev/null +++ b/comfyui-videohelpersuite/tests/audio.json @@ -0,0 +1,313 @@ +{ + "id": "07b812b5-5037-4878-90bc-32d3a1f36619", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 5, + "type": "VHS_VideoCombine", + "pos": [ + 732, + -23 + ], + "size": [ + 210, + 334 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 2 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 3 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/webm", + "pix_fmt": "yuv420p", + "crf": 20, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": {} + } + } + }, + { + "id": 6, + "type": "VHS_VideoCombine", + "pos": [ + 503, + 363 + ], + "size": [ + 210, + 334 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 5 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": {} + } + } + }, + { + "id": 4, + "type": "VHS_LoadVideoPath", + "pos": [ + 29, + 16 + ], + "size": [ + 221.27618408203125, + 413.1552734375 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2, + 4 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 3 + ] + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "input/bigbuckbunny.mp4", + "force_rate": 8, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 30, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "AnimateDiff", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "input/bigbuckbunny.mp4", + "type": "path", + "format": "video/mp4", + "force_rate": 8, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 30, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 7, + "type": "VHS_LoadAudio", + "pos": [ + 83, + 564 + ], + "size": [ + 218.93820190429688, + 126 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 5 + ] + }, + { + "name": "duration", + "type": "FLOAT", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadAudio" + }, + "widgets_values": { + "audio_file": "input/bigbuckbunny.mp4", + "seek_seconds": 0, + "duration": 5 + } + } + ], + "links": [ + [ + 2, + 4, + 0, + 5, + 0, + "IMAGE" + ], + [ + 3, + 4, + 2, + 5, + 1, + "AUDIO" + ], + [ + 4, + 4, + 0, + 6, + 0, + "IMAGE" + ], + [ + 5, + 7, + 0, + 6, + 1, + "AUDIO" + ] + ], + "groups": [], + "config": {}, + "extra": { + "frontendVersion": "1.25.0", + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4, + "tests": { + "5": [{"type": "audio", "key": "nb_read_packets", "value": "177"}], + "6": [{"type": "audio", "key": "nb_read_packets", "value": "176"}], + "length": 2 + } +} \ No newline at end of file diff --git a/comfyui-videohelpersuite/tests/batch4x4.json b/comfyui-videohelpersuite/tests/batch4x4.json new file mode 100644 index 0000000000000000000000000000000000000000..34b9647329642a9e5ca233a60b8bbf2ca8f72aca --- /dev/null +++ b/comfyui-videohelpersuite/tests/batch4x4.json @@ -0,0 +1,535 @@ +{ + "last_node_id": 18, + "last_link_id": 28, + "nodes": [ + { + "id": 16, + "type": "SolidMask", + "pos": [ + 38, + 1066 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 18 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 512, + 512 + ] + }, + { + "id": 15, + "type": "ImageCompositeMasked", + "pos": [ + 412.0800030517579, + 591.4099975585939 + ], + "size": { + "0": 315, + "1": 146 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "destination", + "type": "IMAGE", + "link": 16 + }, + { + "name": "source", + "type": "IMAGE", + "link": 17 + }, + { + "name": "mask", + "type": "MASK", + "link": 18 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 19 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + false + ] + }, + { + "id": 18, + "type": "VHS_BatchManager", + "pos": [ + 329.8000881958008, + -87.40008056640622 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "VHS_BatchManager", + "type": "VHS_BatchManager", + "links": [ + 24, + 25, + 26, + 27 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_BatchManager" + }, + "widgets_values": { + "frames_per_batch": 4, + "count": 2 + } + }, + { + "id": 10, + "type": "VHS_LoadVideoPath", + "pos": [ + 24, + 102 + ], + "size": [ + 320, + 420.7188019966722 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": 25, + "slot_index": 0 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 16, + 22 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 28 + ], + "slot_index": 2, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "input/bigbuckbunny.mp4", + "force_rate": 8, + "force_size": "512x?", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 16, + "skip_first_frames": 64, + "select_every_nth": 1, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 16, + "skip_first_frames": 64, + "force_rate": 8, + "filename": "input/bigbuckbunny.mp4", + "type": "path", + "format": "video/mp4", + "select_every_nth": 1, + "force_size": "512x?" + } + } + } + }, + { + "id": 14, + "type": "VHS_LoadVideoPath", + "pos": [ + 26, + 598 + ], + "size": [ + 320, + 444.7188019966722 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": 26, + "slot_index": 0 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": [], + "slot_index": 2, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "input/bigbuckbunny.mp4", + "force_rate": 8, + "force_size": "Custom Width", + "custom_width": 384, + "custom_height": 512, + "frame_load_cap": 16, + "skip_first_frames": 64, + "select_every_nth": 1, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 16, + "skip_first_frames": 64, + "force_rate": 8, + "filename": "input/bigbuckbunny.mp4", + "type": "path", + "format": "video/mp4", + "select_every_nth": 1, + "force_size": "512x?" + } + } + } + }, + { + "id": 11, + "type": "VHS_VideoCombine", + "pos": [ + 762, + 646 + ], + "size": [ + 320, + 492.75 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 19 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 28 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": 27, + "slot_index": 2 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00005-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 756, + 100 + ], + "size": [ + 320, + 492.75 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 22 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": 24, + "slot_index": 2 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/webm", + "pix_fmt": "yuv420p", + "crf": 20, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00004.webm", + "subfolder": "", + "type": "temp", + "format": "video/webm", + "frame_rate": 8 + } + } + } + } + ], + "links": [ + [ + 16, + 10, + 0, + 15, + 0, + "IMAGE" + ], + [ + 17, + 14, + 0, + 15, + 1, + "IMAGE" + ], + [ + 18, + 16, + 0, + 15, + 2, + "MASK" + ], + [ + 19, + 15, + 0, + 11, + 0, + "IMAGE" + ], + [ + 22, + 10, + 0, + 17, + 0, + "IMAGE" + ], + [ + 24, + 18, + 0, + 17, + 2, + "VHS_BatchManager" + ], + [ + 25, + 18, + 0, + 10, + 0, + "VHS_BatchManager" + ], + [ + 26, + 18, + 0, + 14, + 0, + "VHS_BatchManager" + ], + [ + 27, + 18, + 0, + 11, + 2, + "VHS_BatchManager" + ], + [ + 28, + 10, + 2, + 11, + 1, + "VHS_AUDIO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4, + "tests": { + "17": [{"type": "video", "key": "nb_read_packets", "value": "16"}], + "11": [{"type": "video", "key": "nb_read_packets", "value": "16"}, + {"type": "compare", "filename": "custom_nodes/ComfyUI-VideoHelperSuite/tests/outputs/batch.mp4", "tolerance": 0.02} + ], + "length": 1 + } +} diff --git a/comfyui-videohelpersuite/tests/converted-format-input.json b/comfyui-videohelpersuite/tests/converted-format-input.json new file mode 100644 index 0000000000000000000000000000000000000000..e9722580df540b89a4b9185260a8de180ddd63aa --- /dev/null +++ b/comfyui-videohelpersuite/tests/converted-format-input.json @@ -0,0 +1,253 @@ +{ + "last_node_id": 19, + "last_link_id": 20, + "nodes": [ + { + "id": 18, + "type": "PrimitiveNode", + "pos": [ + 318, + 618 + ], + "size": [ + 210, + 82 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 18 + ], + "widget": { + "name": "crf" + } + } + ], + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 60, + "fixed" + ] + }, + { + "id": 11, + "type": "LoadImage", + "pos": [ + 260.4530029296875, + 233.2003173828125 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 19 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example.png", + "image" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 733.3749389648438, + 338.28924560546875 + ], + "size": [ + 222.91415405273438, + 522.9141845703125 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 19 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + }, + { + "name": "crf", + "type": "INT", + "link": 18, + "widget": { + "name": "crf" + } + }, + { + "name": "pix_fmt", + "type": [ + "yuv420p", + "yuv420p10le" + ], + "link": 20, + "widget": { + "name": "pix_fmt" + } + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p10le", + "crf": 60, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00001.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 8 + }, + "muted": false + } + } + }, + { + "id": 19, + "type": "PrimitiveNode", + "pos": [ + 300, + 760 + ], + "size": [ + 290, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "COMBO", + "type": "COMBO", + "links": [ + 20 + ], + "widget": { + "name": "pix_fmt" + } + } + ], + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + "yuv420p10le", + "fixed", + "" + ] + } + ], + "links": [ + [ + 18, + 18, + 0, + 17, + 4, + "INT" + ], + [ + 19, + 11, + 0, + 17, + 0, + "IMAGE" + ], + [ + 20, + 19, + 0, + 17, + 5, + [ + "yuv420p", + "yuv420p10le" + ] + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 45.8650452880864, + -157.46987175292935 + ] + } + }, + "version": 0.4, + "tests": { + "17": [{"type": "video", "key": "pix_fmt", "value": "yuv420p10le"} + ], + "length": 1 + } +} diff --git a/comfyui-videohelpersuite/tests/converted-input.json b/comfyui-videohelpersuite/tests/converted-input.json new file mode 100644 index 0000000000000000000000000000000000000000..41a93a049ea95ea9ddda92ed3a5152b55581938f --- /dev/null +++ b/comfyui-videohelpersuite/tests/converted-input.json @@ -0,0 +1,307 @@ +{ + "last_node_id": 20, + "last_link_id": 23, + "nodes": [ + { + "id": 18, + "type": "VHS_VideoInfoLoaded", + "pos": [ + 424, + 427 + ], + "size": { + "0": 304.79998779296875, + "1": 106 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "link": 19 + } + ], + "outputs": [ + { + "name": "fps🟦", + "type": "FLOAT", + "links": [ + 20 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count🟦", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "duration🟦", + "type": "FLOAT", + "links": null, + "shape": 3 + }, + { + "name": "width🟦", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "height🟦", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoInfoLoaded" + }, + "widgets_values": {} + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 783, + 223 + ], + "size": [ + 315, + 286 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 21 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "frame_rate", + "type": "FLOAT", + "link": 20, + "widget": { + "name": "frame_rate" + } + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00001.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4" + } + } + } + }, + { + "id": 16, + "type": "VHS_LoadVideoPath", + "pos": [ + 96, + 230 + ], + "size": [ + 315, + 449.5 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "frame_load_cap", + "type": "INT", + "link": 23, + "slot_index": 1, + "widget": { + "name": "frame_load_cap" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 21 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": [ + 19 + ], + "slot_index": 3, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "input/leader.webm", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 64, + "skip_first_frames": 0, + "select_every_nth": 1, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 64, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "input/leader.webm", + "type": "path", + "format": "video/webm", + "select_every_nth": 1 + } + } + } + }, + { + "id": 20, + "type": "PrimitiveNode", + "pos": [ + -240, + 170 + ], + "size": { + "0": 210, + "1": 80 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 23 + ], + "widget": { + "name": "frame_load_cap" + } + } + ], + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 64, + "fixed" + ] + } + ], + "links": [ + [ + 19, + 16, + 3, + 18, + 0, + "VHS_VIDEOINFO" + ], + [ + 20, + 18, + 0, + 17, + 3, + "FLOAT" + ], + [ + 21, + 16, + 0, + 17, + 0, + "IMAGE" + ], + [ + 23, + 20, + 0, + 16, + 1, + "INT" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4, + "tests": { + "17": [{"type": "video", "key": "width", "value": 1440}, + {"type": "video", "key": "height", "value": 1080}, + {"type": "video", "key": "nb_read_packets", "value": "64"} + ], + "length": 1 + } +} diff --git a/comfyui-videohelpersuite/tests/loop.json b/comfyui-videohelpersuite/tests/loop.json new file mode 100644 index 0000000000000000000000000000000000000000..aa5dd4ecdaac29d04b71dbf3bbc8349f2d1210af --- /dev/null +++ b/comfyui-videohelpersuite/tests/loop.json @@ -0,0 +1,178 @@ +{ + "last_node_id": 3, + "last_link_id": 1, + "nodes": [ + { + "id": 1, + "type": "VHS_LoadVideo", + "pos": { + "0": 54, + "1": 89 + }, + "size": [ + 260, + 460 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "leader.webm", + "force_rate": 8, + "force_size": "Disabled", + "custom_width": 304, + "custom_height": 312, + "frame_load_cap": 16, + "skip_first_frames": 1, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 16, + "skip_first_frames": 1, + "force_rate": 8, + "filename": "leader.webm", + "type": "input", + "format": "video/mp4", + "force_size": "410.4x?", + "select_every_nth": 1 + } + } + } + }, + { + "id": 3, + "type": "VHS_VideoCombine", + "pos": { + "0": 629, + "1": 222 + }, + "size": [ + 320, + 550 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 1 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 1, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00005.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + } + ], + "links": [ + [ + 1, + 1, + 0, + 3, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4, + "tests": { + "3": [{"type": "video", "key": "nb_read_packets", "value": "32"}], + "length": 1 + } +} diff --git a/comfyui-videohelpersuite/tests/old-prores.json b/comfyui-videohelpersuite/tests/old-prores.json new file mode 100644 index 0000000000000000000000000000000000000000..17e77c1547663d39b5356ac15d74f807c69dc671 --- /dev/null +++ b/comfyui-videohelpersuite/tests/old-prores.json @@ -0,0 +1,423 @@ +{ + "id": "bc7448ba-118f-4226-95ab-32227992f954", + "revision": 0, + "last_node_id": 6, + "last_link_id": 4, + "nodes": [ + { + "id": 1, + "type": "VHS_LoadVideo", + "pos": [ + 54, + 89 + ], + "size": [ + 245.1999969482422, + 492.7751770019531 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 1, + 2, + 3, + 4 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "leader.webm", + "force_rate": 8, + "custom_width": 512, + "custom_height": 0, + "frame_load_cap": 64, + "skip_first_frames": 1, + "select_every_nth": 1, + "format": "AnimateDiff", + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 64, + "skip_first_frames": 0, + "force_rate": 8, + "filename": "leader.webm", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 4, + "type": "VHS_VideoCombine", + "pos": [ + 630.9500122070312, + 136.90997314453125 + ], + "size": [ + 315, + 497.25 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 2 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/ProRes", + "profile": "2", + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00004.mov", + "subfolder": "", + "type": "temp", + "format": "video/ProRes", + "frame_rate": 8, + "workflow": "AnimateDiff_00004.png" + } + } + } + }, + { + "id": 3, + "type": "VHS_VideoCombine", + "pos": [ + 635.0499267578125, + -407.1000061035156 + ], + "size": [ + 315, + 497.25 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 1 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/ProRes", + "profile": "1", + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00001.mov", + "subfolder": "", + "type": "temp", + "format": "video/ProRes", + "frame_rate": 8, + "workflow": "AnimateDiff_00001.png" + } + } + } + }, + { + "id": 5, + "type": "VHS_VideoCombine", + "pos": [ + 974.6401977539062, + -409.33984375 + ], + "size": [ + 315, + 497.25 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 3 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/ProRes", + "profile": "3", + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00002.mov", + "subfolder": "", + "type": "temp", + "format": "video/ProRes", + "frame_rate": 8, + "workflow": "AnimateDiff_00002.png" + } + } + } + }, + { + "id": 6, + "type": "VHS_VideoCombine", + "pos": [ + 968.7000122070312, + 138.7698974609375 + ], + "size": [ + 315, + 497.25 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/ProRes", + "profile": "4", + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00003.mov", + "subfolder": "", + "type": "temp", + "format": "video/ProRes", + "frame_rate": 8 + } + } + } + } + ], + "links": [ + [ + 1, + 1, + 0, + 3, + 0, + "IMAGE" + ], + [ + 2, + 1, + 0, + 4, + 0, + "IMAGE" + ], + [ + 3, + 1, + 0, + 5, + 0, + "IMAGE" + ], + [ + 4, + 1, + 0, + 6, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "frontendVersion": "1.17.0", + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4, + "tests": { + "6": [{"type": "video", "key": "pix_fmt", "value": "yuv444p12le"}], + "length": 1 + } +} \ No newline at end of file diff --git a/comfyui-videohelpersuite/tests/old-vae-conversion.json b/comfyui-videohelpersuite/tests/old-vae-conversion.json new file mode 100644 index 0000000000000000000000000000000000000000..e751918b6473cae2f38536d8433bc40b2283a3f6 --- /dev/null +++ b/comfyui-videohelpersuite/tests/old-vae-conversion.json @@ -0,0 +1,229 @@ +{ + "last_node_id": 10, + "last_link_id": 9, + "nodes": [ + { + "id": 7, + "type": "VHS_VideoCombine", + "pos": [ + 746, + 309 + ], + "size": [ + 320, + 468.7188019966722 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "LATENT", + "link": 9 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/webm", + "crf": 20, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00001.webm", + "subfolder": "", + "type": "temp", + "format": "video/webm", + "frame_rate": 8 + } + } + } + }, + { + "id": 6, + "type": "VHS_LoadVideoPath", + "pos": [ + 171, + 239 + ], + "size": [ + 320, + 420.7188019966722 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": 7, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 9 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": [], + "slot_index": 2, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "input/bigbuckbunny.mp4", + "force_rate": 8, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 64, + "skip_first_frames": 0, + "select_every_nth": 1, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 64, + "skip_first_frames": 0, + "force_rate": 8, + "filename": "input/bigbuckbunny.mp4", + "type": "path", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 10, + "type": "VAELoader", + "pos": [ + -42, + 88 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 7, + 8 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "taesd" + ] + } + ], + "links": [ + [ + 7, + 10, + 0, + 6, + 1, + "VAE" + ], + [ + 8, + 10, + 0, + 7, + 3, + "VAE" + ], + [ + 9, + 6, + 0, + 7, + 0, + "LATENT" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4, + "tests": { + "7": [{"type": "video", "key": "width", "value": 1920}, + {"type": "video", "key": "height", "value": 1080}, + {"type": "video", "key": "nb_read_packets", "value": "64"} + ], + "length": 1 + } +} diff --git a/comfyui-videohelpersuite/tests/simple.json b/comfyui-videohelpersuite/tests/simple.json new file mode 100644 index 0000000000000000000000000000000000000000..4d7f8ddd68a24cc90519ecd127daae9d8ab81b85 --- /dev/null +++ b/comfyui-videohelpersuite/tests/simple.json @@ -0,0 +1,134 @@ +{ + "last_node_id": 3, + "last_link_id": 1, + "nodes": [ + { + "id": 1, + "type": "VHS_LoadVideo", + "pos": [ + 54, + 89 + ], + "size": [ + 235.1999969482422, + 384.56999829610186 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "leader.webm", + "force_rate": 8, + "force_size": "Custom Width", + "custom_width": 304, + "custom_height": 312, + "frame_load_cap": 16, + "skip_first_frames": 1, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "leader.webm", + "type": "input", + "format": "video/mp4", + "force_size": "410.4x?", + "select_every_nth": 1 + } + } + } + }, + { + "id": 3, + "type": "VHS_VideoCombine", + "pos": [ + 629, + 222 + ], + "size": { + "0": 315, + "1": 250 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 1 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/webm", + "pingpong": false, + "save_image": false, + "crf": 20, + "save_metadata": false, + "audio_file": "", + "videopreview": { + "hidden": false, + "paused": false + } + } + } + ], + "links": [ + [ + 1, + 1, + 0, + 3, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4, + "tests": { + "3": [{"type": "video", "key": "width", "value": 304}, + {"type": "video", "key": "height", "value": 232}, + {"type": "compare", "filename": "custom_nodes/ComfyUI-VideoHelperSuite/tests/outputs/simple.webm", "tolerance": 0.02} + ], + "length": 1 + } +} diff --git a/comfyui-videohelpersuite/video_formats/16bit-png.json b/comfyui-videohelpersuite/video_formats/16bit-png.json new file mode 100644 index 0000000000000000000000000000000000000000..b768bdbcfe8950cb7bde37d02444f0191ad29f51 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/16bit-png.json @@ -0,0 +1,9 @@ +{ + "main_pass": + [ + "-n", + "-pix_fmt", "rgba64" + ], + "input_color_depth": "16bit", + "extension": "%03d.png" +} diff --git a/comfyui-videohelpersuite/video_formats/8bit-png.json b/comfyui-videohelpersuite/video_formats/8bit-png.json new file mode 100644 index 0000000000000000000000000000000000000000..88b27a75f9c979f18979ec46da8c8843a82e0759 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/8bit-png.json @@ -0,0 +1,7 @@ +{ + "main_pass": + [ + "-n" + ], + "extension": "%03d.png" +} diff --git a/comfyui-videohelpersuite/video_formats/ProRes.json b/comfyui-videohelpersuite/video_formats/ProRes.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd3acf931c5ab3486197c18ba88a3959d3e1d6 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/ProRes.json @@ -0,0 +1,22 @@ +{ + "main_pass": + [ + "-n", "-c:v", "prores_ks", + "-profile:v", [["$profile"]], + ["profile", { + "lt": [[]], "1": [[]], "standard": [[]], "2": [[]], "hq": [[]], "3": [[]], + "4": ["has_alpha", {"True": [["-pix_fmt", "yuva444p10le"]], + "False": [["-pix_fmt", "yuv444p10le"]]}], + "4444": ["has_alpha", {"True": [["-pix_fmt", "yuva444p10le"]], + "False": [["-pix_fmt", "yuv444p10le"]]}], + "4444xq": ["has_alpha", {"True": [["-pix_fmt", "yuva444p10le"]], + "False": [["-pix_fmt", "yuv444p10le"]]}] + }], + "-vf", "scale=out_color_matrix=bt709", + "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "pcm_s16le"], + "extension": "mov", + "extra_widgets": [["profile", ["lt", "standard", "hq", "4444", "4444xq"], {"default": "hq"}]] +} diff --git a/comfyui-videohelpersuite/video_formats/av1-webm.json b/comfyui-videohelpersuite/video_formats/av1-webm.json new file mode 100644 index 0000000000000000000000000000000000000000..659ea166fb140e18109f09f3d5710363af9c2294 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/av1-webm.json @@ -0,0 +1,16 @@ +{ + "main_pass": + [ + "-n", "-c:v", "libsvtav1", + "-pix_fmt", ["pix_fmt", ["yuv420p10le", "yuv420p"]], + "-crf", ["crf","INT", {"default": 23, "min": 0, "max": 100, "step": 1}], + "-vf", "scale=out_color_matrix=bt709", + "-color_range", "tv", "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "libopus"], + "input_color_depth": ["input_color_depth", ["8bit", "16bit"]], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "extension": "webm", + "environment": {"SVT_LOG": "1"} +} diff --git a/comfyui-videohelpersuite/video_formats/ffmpeg-gif.json b/comfyui-videohelpersuite/video_formats/ffmpeg-gif.json new file mode 100644 index 0000000000000000000000000000000000000000..3ca0865033c7dabc06176c737652f375e7badb27 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/ffmpeg-gif.json @@ -0,0 +1,8 @@ +{ + "main_pass": + [ + "-n", + "-filter_complex", ["dither", ["bayer", "heckbert", "floyd_steinberg", "sierra2", "sierra2_4a", "sierra3", "burkes", "atkinson", "none"], {"default": "sierra2_4a"}, "[0:v] split [a][b]; [a] palettegen=reserve_transparent=on:transparency_color=ffffff [p]; [b][p] paletteuse=dither=$val"] + ], + "extension": "gif" +} diff --git a/comfyui-videohelpersuite/video_formats/ffv1-mkv.json b/comfyui-videohelpersuite/video_formats/ffv1-mkv.json new file mode 100644 index 0000000000000000000000000000000000000000..3248e5fc6425feeb1d55ccdb8bd8d49cb0e3401c --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/ffv1-mkv.json @@ -0,0 +1,18 @@ +{ + "main_pass": [ + "-n", + "-c:v", "ffv1", + "-level", ["level", ["0", "1", "3"], {"default": "3"}], + "-coder", ["coder", ["0", "1", "2"], {"default": "1"}], + "-context", ["context", ["0", "1"], {"default": "1"}], + "-g", ["gop_size", "INT", {"default": 1, "min": 1, "max": 300, "step": 1}], + "-slices", ["slices", ["4", "6", "9", "12", "16", "20", "24", "30"], {"default": "16"}], + "-slicecrc", ["slicecrc", ["0", "1"], {"default": "1"}], + "-pix_fmt", ["pix_fmt", ["rgba64le", "bgra", "yuv420p", "yuv422p", "yuv444p", "yuva420p", "yuva422p", "yuva444p", "yuv420p10le", "yuv422p10le", "yuv444p10le", "yuv420p12le", "yuv422p12le", "yuv444p12le", "yuv420p14le", "yuv422p14le", "yuv444p14le", "yuv420p16le", "yuv422p16le", "yuv444p16le", "gray", "gray10le", "gray12le", "gray16le"], {"default": "rgba64le"}] + ], + "audio_pass": ["-c:a", "flac"], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "trim_to_audio": ["trim_to_audio", "BOOLEAN", {"default": false}], + "input_color_depth": "16bit", + "extension": "mkv" +} diff --git a/comfyui-videohelpersuite/video_formats/gifski.json b/comfyui-videohelpersuite/video_formats/gifski.json new file mode 100644 index 0000000000000000000000000000000000000000..c96b358ab0ae59dbd97c52a699c8ee9825c5d351 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/gifski.json @@ -0,0 +1,12 @@ +{ + "main_pass": + [ + "-pix_fmt", "yuv444p", + "-vf", "scale=out_color_matrix=bt709:out_range=pc", + "-color_range", "pc" + ], + "extension": "gif", + "gifski_pass": [ + "-Q", ["quality","INT", {"default": 90, "min": 1, "max": 100, "step": 1}] + ] +} diff --git a/comfyui-videohelpersuite/video_formats/h264-mp4.json b/comfyui-videohelpersuite/video_formats/h264-mp4.json new file mode 100644 index 0000000000000000000000000000000000000000..89d290c92cd44674e2fe6dca8d0a240d6bf9695b --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/h264-mp4.json @@ -0,0 +1,15 @@ +{ + "main_pass": + [ + "-n", "-c:v", "libx264", + "-pix_fmt", ["pix_fmt", ["yuv420p", "yuv420p10le"]], + "-crf", ["crf","INT", {"default": 19, "min": 0, "max": 100, "step": 1}], + "-vf", "scale=out_color_matrix=bt709", + "-color_range", "tv", "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "aac"], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "trim_to_audio": ["trim_to_audio", "BOOLEAN", {"default": false}], + "extension": "mp4" +} diff --git a/comfyui-videohelpersuite/video_formats/h265-mp4.json b/comfyui-videohelpersuite/video_formats/h265-mp4.json new file mode 100644 index 0000000000000000000000000000000000000000..b5d0150bf9cdb21f396f026a0d7721626c698822 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/h265-mp4.json @@ -0,0 +1,17 @@ +{ + "main_pass": + [ + "-n", "-c:v", "libx265", + "-vtag", "hvc1", + "-pix_fmt", ["pix_fmt", ["yuv420p10le", "yuv420p"]], + "-crf", ["crf","INT", {"default": 22, "min": 0, "max": 100, "step": 1}], + "-preset", "medium", + "-x265-params", "log-level=quiet", + "-vf", "scale=out_color_matrix=bt709", + "-color_range", "tv", "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "aac"], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "extension": "mp4" +} diff --git a/comfyui-videohelpersuite/video_formats/nvenc_av1-mp4.json b/comfyui-videohelpersuite/video_formats/nvenc_av1-mp4.json new file mode 100644 index 0000000000000000000000000000000000000000..0996c65b3f586897a2b131997ff1faaa6a4b828e --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/nvenc_av1-mp4.json @@ -0,0 +1,15 @@ +{ + "main_pass": + [ + "-n", "-c:v", "av1_nvenc", + "-pix_fmt", ["pix_fmt", ["yuv420p", "p010le"]], + "-vf", "scale=out_color_matrix=bt709", + "-color_range", "tv", "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "aac"], + "bitrate": ["bitrate","INT", {"default": 10, "min": 1, "max": 999, "step": 1 }], + "megabit": ["megabit","BOOLEAN", {"default": true}], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "extension": "mp4" +} diff --git a/comfyui-videohelpersuite/video_formats/nvenc_h264-mp4.json b/comfyui-videohelpersuite/video_formats/nvenc_h264-mp4.json new file mode 100644 index 0000000000000000000000000000000000000000..1c39913ab6419fb2f7b1c0a6241b7fea1cc2cb11 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/nvenc_h264-mp4.json @@ -0,0 +1,15 @@ +{ + "main_pass": + [ + "-n", "-c:v", "h264_nvenc", + "-pix_fmt", ["pix_fmt", ["yuv420p", "p010le"]], + "-vf", "scale=out_color_matrix=bt709", + "-color_range", "tv", "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "aac"], + "bitrate": ["bitrate","INT", {"default": 10, "min": 1, "max": 999, "step": 1 }], + "megabit": ["megabit","BOOLEAN", {"default": true}], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "extension": "mp4" +} diff --git a/comfyui-videohelpersuite/video_formats/nvenc_hevc-mp4.json b/comfyui-videohelpersuite/video_formats/nvenc_hevc-mp4.json new file mode 100644 index 0000000000000000000000000000000000000000..2ec6f4169d02466287026e4989b930ec3b64c200 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/nvenc_hevc-mp4.json @@ -0,0 +1,16 @@ +{ + "main_pass": + [ + "-n", "-c:v", "hevc_nvenc", + "-vtag", "hvc1", + "-pix_fmt", ["pix_fmt", ["yuv420p", "p010le"]], + "-vf", "scale=out_color_matrix=bt709", + "-color_range", "tv", "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "aac"], + "bitrate": ["bitrate","INT", {"default": 10, "min": 1, "max": 999, "step": 1 }], + "megabit": ["megabit","BOOLEAN", {"default": true}], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "extension": "mp4" +} diff --git a/comfyui-videohelpersuite/video_formats/webm.json b/comfyui-videohelpersuite/video_formats/webm.json new file mode 100644 index 0000000000000000000000000000000000000000..6323f619ad2850ba60cd5e31b32cdfc060e030c9 --- /dev/null +++ b/comfyui-videohelpersuite/video_formats/webm.json @@ -0,0 +1,16 @@ +{ + "main_pass": + [ + "-n", + "-pix_fmt", ["pix_fmt",["yuv420p","yuva420p"]], + "-crf", ["crf","INT", {"default": 20, "min": 0, "max": 100, "step": 1}], + "-b:v", "0", + "-vf", "scale=out_color_matrix=bt709", + "-color_range", "tv", "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709" + ], + "fake_trc": "bt709", + "audio_pass": ["-c:a", "libvorbis"], + "save_metadata": ["save_metadata", "BOOLEAN", {"default": true}], + "trim_to_audio": ["trim_to_audio", "BOOLEAN", {"default": false}], + "extension": "webm" +} diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/batched_nodes.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/batched_nodes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b52ed3df8e79da1def69410eebe53308e2fca366 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/batched_nodes.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/documentation.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/documentation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52cbe23f49b8f7a78ba6d4d84f81dfa480a5b9d6 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/documentation.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/image_latent_nodes.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/image_latent_nodes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fad2de1a9621d586f07ea2f512eda1e7d3e6fe0 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/image_latent_nodes.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/latent_preview.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/latent_preview.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b2e8a9a4c956d962338862b9f0a6c8f201d27b03 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/latent_preview.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/load_images_nodes.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/load_images_nodes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7bc16d2a0937d6363c216d2623be17d6790a0f61 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/load_images_nodes.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/load_video_nodes.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/load_video_nodes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de638277673f095b343911d02cddbdb3fc43eddc Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/load_video_nodes.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/logger.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/logger.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f51a154bc29b364cf6326c5c06595103abe8bc27 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/logger.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/nodes.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/nodes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0bb54149986ae5728153a6b2cd3f1debc377c178 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/nodes.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/server.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/server.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f0bb1b6857c830101c47d280e2c868a63f4c7b3 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/server.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/__pycache__/utils.cpython-312.pyc b/comfyui-videohelpersuite/videohelpersuite/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb22fe7b550916825f6e076b2c376b57fead5896 Binary files /dev/null and b/comfyui-videohelpersuite/videohelpersuite/__pycache__/utils.cpython-312.pyc differ diff --git a/comfyui-videohelpersuite/videohelpersuite/batched_nodes.py b/comfyui-videohelpersuite/videohelpersuite/batched_nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..624bb84f73cce1d8e51abf4fef0b80367ad4b9a3 --- /dev/null +++ b/comfyui-videohelpersuite/videohelpersuite/batched_nodes.py @@ -0,0 +1,56 @@ +import torch +from nodes import VAEEncode +from comfy.utils import ProgressBar + + +class VAEDecodeBatched: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "samples": ("LATENT", ), + "vae": ("VAE", ), + "per_batch": ("INT", {"default": 16, "min": 1}) + } + } + + CATEGORY = "Video Helper Suite 🎥🅥🅗🅢/batched nodes" + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "decode" + + def decode(self, vae, samples, per_batch): + decoded = [] + pbar = ProgressBar(samples["samples"].shape[0]) + for start_idx in range(0, samples["samples"].shape[0], per_batch): + decoded.append(vae.decode(samples["samples"][start_idx:start_idx+per_batch])) + pbar.update(per_batch) + return (torch.cat(decoded, dim=0), ) + + +class VAEEncodeBatched: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "pixels": ("IMAGE", ), "vae": ("VAE", ), + "per_batch": ("INT", {"default": 16, "min": 1}) + } + } + + CATEGORY = "Video Helper Suite 🎥🅥🅗🅢/batched nodes" + + RETURN_TYPES = ("LATENT",) + FUNCTION = "encode" + + def encode(self, vae, pixels, per_batch): + t = [] + pbar = ProgressBar(pixels.shape[0]) + for start_idx in range(0, pixels.shape[0], per_batch): + try: + sub_pixels = vae.vae_encode_crop_pixels(pixels[start_idx:start_idx+per_batch]) + except: + sub_pixels = VAEEncode.vae_encode_crop_pixels(pixels[start_idx:start_idx+per_batch]) + t.append(vae.encode(sub_pixels[:,:,:,:3])) + pbar.update(per_batch) + return ({"samples": torch.cat(t, dim=0)}, ) diff --git a/comfyui-videohelpersuite/videohelpersuite/documentation.py b/comfyui-videohelpersuite/videohelpersuite/documentation.py new file mode 100644 index 0000000000000000000000000000000000000000..857de6c6a5b4a3917f49d0b1c5c142f274203f22 --- /dev/null +++ b/comfyui-videohelpersuite/videohelpersuite/documentation.py @@ -0,0 +1,616 @@ +from .logger import logger + +def image(src): + return f'' +def video(src): + return f'